From 1f30c49f38ed074abdb3e36ec48d69c0f281b413 Mon Sep 17 00:00:00 2001 From: wing-anara Date: Wed, 14 Oct 2026 10:09:40 +0000 Subject: [PATCH 1/4] Render agent visuals natively from sandboxed Luau scripts show_visual now takes a Luau script instead of an HTML page. The script builds its data and state once and returns a view function whose ui.* tree (layout, text, stats, tables, charts, tabs, toggles, sliders, buttons, links, 2D canvases, 3D scenes) Zeron renders with its own GPUI elements, so a visual scrolls, clips, blurs under the glass composer and takes input like any transcript row on every platform. zeron-visual is the GPUI-free runtime: a Luau sandbox with no I/O, a memory cap and per-call time budgets, seeded randomness, strict props whose style rules are the API (theme color tokens, the transcript's type scale, no fills or borders), canvas drawing commands, and 3D scene projection. The engine runs every script through check() before showing it and returns line-numbered errors to the agent. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 68 +- Cargo.toml | 2 + crates/doc/src/parts.rs | 22 + crates/doc/src/schema.rs | 53 + crates/engine/Cargo.toml | 1 + crates/engine/src/rpc.rs | 66 +- crates/engine/src/sessions.rs | 9 + crates/engine/src/uploads.rs | 14 + crates/engine/tests/e2e.rs | 138 ++ crates/mcp/Cargo.toml | 3 + crates/mcp/src/jsonrpc.rs | 9 +- crates/mcp/src/tools.rs | 206 ++- crates/mcp/src/transcript.rs | 1 + crates/proto/src/agent.rs | 14 + crates/rpc/src/lib.rs | 3 + crates/ui/Cargo.toml | 1 + crates/ui/src/lib.rs | 1 + crates/ui/src/transcript.rs | 85 ++ crates/ui/src/visuals.rs | 2282 ++++++++++++++++++++++++++++++++ crates/visual/Cargo.toml | 11 + crates/visual/src/draw.rs | 421 ++++++ crates/visual/src/lib.rs | 536 ++++++++ crates/visual/src/node.rs | 1002 ++++++++++++++ crates/visual/src/prelude.luau | 99 ++ crates/visual/src/scene.rs | 982 ++++++++++++++ crates/visual/src/style.rs | 267 ++++ crates/visual/src/tests.rs | 278 ++++ docs/mcp.md | 45 + 28 files changed, 6606 insertions(+), 13 deletions(-) create mode 100644 crates/ui/src/visuals.rs create mode 100644 crates/visual/Cargo.toml create mode 100644 crates/visual/src/draw.rs create mode 100644 crates/visual/src/lib.rs create mode 100644 crates/visual/src/node.rs create mode 100644 crates/visual/src/prelude.luau create mode 100644 crates/visual/src/scene.rs create mode 100644 crates/visual/src/style.rs create mode 100644 crates/visual/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 045f5d9477..5aa756f94e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,7 +961,7 @@ dependencies = [ "bitflags 2.13.1", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.12.1", "log", "prettyplease", "proc-macro2", @@ -2852,7 +2852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4988,7 +4988,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5620,6 +5620,15 @@ dependencies = [ "serde_repr", ] +[[package]] +name = "luau0-src" +version = "0.22.0+luau740" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea012e8a8aaff2f7b6c788fad41a60630003b6a0932bcf045f7d14952c9bd31" +dependencies = [ + "cc", +] + [[package]] name = "lyon" version = "1.0.19" @@ -5923,6 +5932,34 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mlua" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7ae6b65bf54b7fdd4cf5b6f44f60862350ebf3fe3779b3a9cc005e7fa14f81" +dependencies = [ + "bstr", + "either", + "libc", + "mlua-sys", + "num-traits", + "parking_lot", + "rustc-hash 2.1.3", +] + +[[package]] +name = "mlua-sys" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1631cda17bd6486410ab751d91f5fd4484f441589278cdad879eb3afc721a285" +dependencies = [ + "cc", + "cfg-if", + "libc", + "luau0-src", + "pkg-config", +] + [[package]] name = "monostate" version = "0.1.18" @@ -6368,7 +6405,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.5.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8015,7 +8052,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8028,7 +8065,7 @@ dependencies = [ "libc", "log", "socket2", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8985,7 +9022,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -8998,7 +9035,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10263,7 +10300,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -12260,7 +12297,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13565,6 +13602,7 @@ dependencies = [ "zeron-rpc", "zeron-sync", "zeron-update", + "zeron-visual", "zip", ] @@ -13619,6 +13657,7 @@ dependencies = [ "zeron-doc", "zeron-proto", "zeron-rpc", + "zeron-visual", ] [[package]] @@ -13867,6 +13906,7 @@ dependencies = [ "zeron-theme", "zeron-update", "zeron-veil", + "zeron-visual", "zeron-voice", "zeron-voice-media", "zeron-voice-session", @@ -13892,6 +13932,14 @@ dependencies = [ name = "zeron-veil" version = "0.2.112" +[[package]] +name = "zeron-visual" +version = "0.2.112" +dependencies = [ + "mlua", + "thiserror 2.0.20", +] + [[package]] name = "zeron-voice" version = "0.2.112" diff --git a/Cargo.toml b/Cargo.toml index f73e255faa..33127f3a5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/client", "crates/mcp", "crates/text", + "crates/visual", "apps/zeron", ] @@ -54,6 +55,7 @@ zeron-client = { path = "crates/client" } zeron-theme = { path = "crates/theme" } zeron-mcp = { path = "crates/mcp" } zeron-text = { path = "crates/text" } +zeron-visual = { path = "crates/visual" } # CRDT loro = "1.13" diff --git a/crates/doc/src/parts.rs b/crates/doc/src/parts.rs index 871ff83ac5..09c28afbf3 100644 --- a/crates/doc/src/parts.rs +++ b/crates/doc/src/parts.rs @@ -134,6 +134,14 @@ pub enum MessagePart { name: String, mime_type: String, }, + /// An agent-authored visual (`show_visual`): a Luau script stored on the + /// host at `path`, rendered natively. Old desktop builds' unknown-kind + /// fallback yields an empty text part (invisible); iOS drops unknown kinds. + Visual { + id: String, + path: String, + title: String, + }, /// Model thinking. Carries its body in a dedicated doc field (`reasoning`, /// never `text`) so pre-reasoning desktop builds — whose unknown-kind /// fallback renders `text` as prose — degrade to an invisible empty text @@ -225,6 +233,7 @@ impl MessagePart { match self { MessagePart::Text { id, .. } | MessagePart::Image { id, .. } + | MessagePart::Visual { id, .. } | MessagePart::Reasoning { id, .. } | MessagePart::Tool { id, .. } | MessagePart::Input { id, .. } @@ -261,6 +270,7 @@ impl MessagePart { name, mime_type, } => id.len() + path.len() + name.len() + mime_type.len(), + MessagePart::Visual { id, path, title } => id.len() + path.len() + title.len(), MessagePart::Error { message, .. } => message.len(), MessagePart::Fork { source_chat_id, @@ -318,6 +328,18 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { out.push(image); } } + AgentEvent::Visual { id, path, title } => { + let visual = MessagePart::Visual { + id: id.clone(), + path: path.clone(), + title: title.clone(), + }; + if let Some(existing) = out.iter_mut().find(|p| p.id() == id) { + *existing = visual; + } else { + out.push(visual); + } + } AgentEvent::ReasoningDelta { text } => { // Empty deltas are liveness heartbeats (redacted thinking) — the // engine drops them before the fold, but stay tolerant here too. diff --git a/crates/doc/src/schema.rs b/crates/doc/src/schema.rs index bc68b3bf8e..7efaa69beb 100644 --- a/crates/doc/src/schema.rs +++ b/crates/doc/src/schema.rs @@ -144,6 +144,14 @@ fn to_doc_part(part: &MessagePart) -> Result { mime_type: Some(mime_type.clone()), ..Default::default() }, + // The script path and title ride `path` and `name`. + MessagePart::Visual { id, path, title } => DocPartJson { + id: id.clone(), + kind: "visual".into(), + path: Some(path.clone()), + name: Some(title.clone()), + ..Default::default() + }, MessagePart::Text { id, text } => DocPartJson { id: id.clone(), kind: "text".into(), @@ -231,6 +239,17 @@ fn to_doc_part(part: &MessagePart) -> Result { fn from_doc_part(p: DocPartJson) -> MessagePart { match p.kind.as_str() { "image" => image_part(p.id, p.path, p.name, p.mime_type), + "visual" => match p.path.filter(|path| !path.is_empty()) { + Some(path) => MessagePart::Visual { + id: p.id, + path, + title: p.name.unwrap_or_default(), + }, + None => MessagePart::Error { + id: p.id, + message: "Visual unavailable".into(), + }, + }, "tool" => match p.call.and_then(|c| serde_json::from_value(c).ok()) { Some(call) => MessagePart::Tool { id: p.id, @@ -1519,6 +1538,40 @@ mod tests { } } + #[test] + fn visual_round_trips_and_degrades_without_a_path() { + let doc = SessionDoc::init("visuals").unwrap(); + let mut writer = SegmentWriter::begin(&doc, "a", "owner", 1).unwrap(); + let mut parts = vec![]; + fold_event_into_parts( + &mut parts, + &AgentEvent::Visual { + id: "v:1".into(), + path: "/uploads/v-1-visual.luau".into(), + title: "Latency by region".into(), + }, + ); + writer.sync(&parts).unwrap(); + let imported = LoroDoc::new(); + imported.import(&doc.export_snapshot().unwrap()).unwrap(); + let loaded = SessionDoc::from_doc(imported); + assert_eq!(loaded.read_entries().unwrap()[0].parts, parts); + assert!(matches!( + from_doc_part( + serde_json::from_value(serde_json::json!({"kind":"visual", "id":"v", "name":"t"})) + .unwrap() + ), + MessagePart::Error { .. } + )); + assert!(matches!( + from_doc_part( + serde_json::from_value(serde_json::json!({"kind":"visual", "id":"v", "path":"/p"})) + .unwrap() + ), + MessagePart::Visual { .. } + )); + } + fn user_entry(id: &str, text: &str) -> SessionMessageEntry { SessionMessageEntry { id: id.into(), diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 23161ba8d8..2a7c5e8820 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -13,6 +13,7 @@ zeron-sync.workspace = true zeron-harness.workspace = true zeron-rpc.workspace = true zeron-update.workspace = true +zeron-visual.workspace = true loro.workspace = true tokio.workspace = true futures.workspace = true diff --git a/crates/engine/src/rpc.rs b/crates/engine/src/rpc.rs index 8b8c775f9d..93bb4d0047 100644 --- a/crates/engine/src/rpc.rs +++ b/crates/engine/src/rpc.rs @@ -63,7 +63,7 @@ use tokio::sync::watch; use zeron_doc::{MessagePart, SessionCommandPayload}; use zeron_proto::{ - ChatConfig, CreateWorktreeOutcome, EngineInfo, HarnessId, HarnessUpdatePolicy, + AgentEvent, ChatConfig, CreateWorktreeOutcome, EngineInfo, HarnessId, HarnessUpdatePolicy, ProjectActionDraft, Space, ToolCall, WorkspaceScope, }; use zeron_rpc::{ @@ -501,6 +501,16 @@ struct ReadAttachmentChunkParams { offset: u64, } +const _: () = assert!(zeron_proto::VISUAL_MAX_BYTES == zeron_visual::MAX_SCRIPT_BYTES); + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PublishVisualParams { + chat_id: String, + title: String, + script: String, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct FetchToolBlobParams { @@ -3691,6 +3701,60 @@ impl RpcService for EngineRpc { .map_err(|e| RpcError::Failed(e.to_string()))?; RpcReply::value(&chunk) } + methods::PUBLISH_VISUAL => { + let p: PublishVisualParams = parse_params(params)?; + let title: String = p + .title + .trim() + .chars() + .take(zeron_proto::VISUAL_MAX_TITLE_CHARS) + .collect(); + if title.is_empty() { + return Err(RpcError::BadParams("title is empty".into())); + } + if p.script.trim().is_empty() || p.script.len() > zeron_proto::VISUAL_MAX_BYTES { + return Err(RpcError::BadParams(format!( + "script must be 1 to {} bytes", + zeron_proto::VISUAL_MAX_BYTES + ))); + } + let id = format!("v:{}", crate::new_id()); + // Run the script the way the desktop will (setup, render, + // every draw, every control) before anyone sees it, so the + // agent gets its errors with line numbers and can fix them. + let uploads = self.uploads.clone(); + let (stored_id, script) = (id.clone(), p.script); + let (path, report) = tokio::task::spawn_blocking(move || { + let checked = zeron_visual::check(&script, zeron_visual::seed_for(&stored_id)); + let report = checked + .map_err(|e| RpcError::BadParams(format!("visual script failed: {e}")))?; + let path = uploads + .store_visual(&stored_id, &script) + .map_err(|e| RpcError::Failed(e.to_string()))?; + Ok::<_, RpcError>((path, report)) + }) + .await + .map_err(|e| RpcError::Failed(e.to_string()))??; + let event = AgentEvent::Visual { + id: id.clone(), + path: path.clone(), + title: title.clone(), + }; + if !self.sessions.inject_event(&p.chat_id, event) { + let _ = std::fs::remove_file(&path); + return Err(RpcError::Failed( + "This chat has no running turn on this device; show_visual only works \ + from inside a turn." + .into(), + )); + } + RpcReply::value(&serde_json::json!({ + "id": id, + "title": title, + "checkedInteractions": report.interactions, + "printed": report.logs, + })) + } methods::FETCH_TOOL_BLOB => { let p: FetchToolBlobParams = parse_params(params)?; let text = self diff --git a/crates/engine/src/sessions.rs b/crates/engine/src/sessions.rs index 9add550662..6dfc5254fa 100644 --- a/crates/engine/src/sessions.rs +++ b/crates/engine/src/sessions.rs @@ -1120,6 +1120,15 @@ impl SessionsEngine { Ok(true) } + /// Fold an engine-made event (a `show_visual` page) into the chat's live + /// run, after whatever the harness streamed so far. Returns `false` when + /// the chat has no run on this device. + pub fn inject_event(&self, chat_id: &str, event: AgentEvent) -> bool { + lock(&self.inner.runs) + .get(chat_id) + .is_some_and(|h| h.engine_tx.send(event).is_ok()) + } + /// Resolve a pending `request_input` question set. Returns `false` when no such /// request is pending (unknown id, or the run already settled). pub fn respond_input( diff --git a/crates/engine/src/uploads.rs b/crates/engine/src/uploads.rs index a3aa7a3690..fdc4eaf178 100644 --- a/crates/engine/src/uploads.rs +++ b/crates/engine/src/uploads.rs @@ -879,6 +879,20 @@ fn open_generated_file( } impl Uploads { + /// Store an agent's `show_visual` script and return its absolute path. + /// It lives beside other attachments, so clients read it the same way. + pub fn store_visual(&self, id: &str, script: &str) -> Result { + use std::io::Write; + std::fs::create_dir_all(self.dir())?; + let path = self.dir().join(format!("{}-visual.luau", sanitize(id))); + let mut temporary = tempfile::NamedTempFile::new_in(self.dir())?; + temporary.write_all(script.as_bytes())?; + temporary + .persist(&path) + .map_err(|err| EngineError::Other(err.to_string()))?; + Ok(path.to_string_lossy().to_string()) + } + /// Copy a Codex-owned raster into the active profile without exposing the /// original path to the journal, document, RPC readers, or relay. pub fn import_generated_image( diff --git a/crates/engine/tests/e2e.rs b/crates/engine/tests/e2e.rs index 827ab515bc..b089ff9039 100644 --- a/crates/engine/tests/e2e.rs +++ b/crates/engine/tests/e2e.rs @@ -3363,3 +3363,141 @@ async fn session_row_counts_running_subagents_until_the_run_ends() { "an ended run leaves no subagents counted" ); } + +/// `show_visual` end to end below the MCP server: the RPC checks the script, +/// stores it, and folds a visual part into the chat's live turn, after what +/// the harness streamed; a broken script or no live turn is refused and +/// stores nothing. +#[tokio::test] +async fn publish_visual_lands_in_the_live_turn() { + struct HoldsTurn { + events: std::sync::Mutex>>, + } + #[async_trait] + impl Harness for HoldsTurn { + fn id(&self) -> HarnessId { + HarnessId::Mock + } + fn display_name(&self) -> &str { + "Held turn" + } + fn supports_steering(&self) -> bool { + true + } + fn steering_mode(&self) -> SteeringMode { + SteeringMode::StepBoundary + } + fn reasoning_levels(&self) -> &[ReasoningLevel] { + &[] + } + async fn models(&self) -> Result, HarnessError> { + Ok(vec![]) + } + async fn run( + &self, + request: RunRequest, + controls: RunControls, + ) -> Result>, HarnessError> { + let events = self.events.lock().unwrap().take(); + match events { + Some(events) => Ok(futures::stream::unfold(events, |mut rx| async move { + rx.recv().await.map(|event| (Ok(event), rx)) + }) + .boxed()), + None => { + MockHarness { + script: mock_script(), + } + .run(request, controls) + .await + } + } + } + } + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let dir = tempfile::tempdir().unwrap(); + let core = assemble( + dir.path(), + Arc::new(HoldsTurn { + events: std::sync::Mutex::new(Some(rx)), + }), + ); + let client = zeron_rpc::memory_client(core.rpc_service()); + const SCRIPT: &str = r#"return ui.text { "chart", size = "lg" }"#; + let publish = |chat: &str, script: &str| { + client.call( + zeron_rpc::methods::PUBLISH_VISUAL, + serde_json::json!({ + "chatId": chat, + "title": " Latency ", + "script": script, + }), + ) + }; + assert!(publish(CHAT, SCRIPT).await.is_err(), "no live turn yet"); + + core.sessions + .dispatch(CHAT, HarnessId::Mock, run_request("chart please"), None) + .await + .unwrap(); + tx.send(mock_script()[0].clone()).unwrap(); + tx.send(AgentEvent::TextDelta { + text: "Building it.".into(), + }) + .unwrap(); + wait_for(|| core.sessions.turn_in_flight(CHAT), "turn starts").await; + let broken = publish(CHAT, "return ui.text { 'x', color = '#f00' }") + .await + .unwrap_err() + .to_string(); + assert!(broken.contains("theme tokens"), "{broken}"); + let shown = publish(CHAT, SCRIPT).await.expect("PublishVisual"); + assert_eq!(shown["title"], "Latency"); + tx.send(AgentEvent::TextDelta { + text: "Uploads dominate.".into(), + }) + .unwrap(); + tx.send(done(DoneStatus::Completed)).unwrap(); + wait_for( + || { + entries(&core).iter().any(|e| { + e.parts + .iter() + .any(|p| matches!(p, MessagePart::Visual { .. })) + }) && !core.sessions.turn_in_flight(CHAT) + }, + "visual folded", + ) + .await; + let parts: Vec = entries(&core) + .into_iter() + .filter(|e| e.role == MessageRole::Assistant) + .flat_map(|e| e.parts) + .collect(); + let at = parts + .iter() + .position(|p| matches!(p, MessagePart::Visual { .. })) + .unwrap(); + let MessagePart::Visual { path, title, .. } = &parts[at] else { + unreachable!() + }; + assert_eq!(title, "Latency"); + assert_eq!(std::fs::read_to_string(path).unwrap(), SCRIPT); + assert_eq!( + parts + .iter() + .filter(|p| matches!(p, MessagePart::Visual { .. })) + .count(), + 1, + "the broken script left no part" + ); + assert!( + matches!(&parts[at - 1], MessagePart::Text { text, .. } if text.ends_with("Building it.")), + "the page follows what streamed before it: {parts:?}" + ); + assert!( + matches!(&parts[at + 1], MessagePart::Text { text, .. } if text == "Uploads dominate."), + "{parts:?}" + ); + core.shutdown().await; +} diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml index 0442ceaf71..ca060c2b72 100644 --- a/crates/mcp/Cargo.toml +++ b/crates/mcp/Cargo.toml @@ -18,3 +18,6 @@ uuid.workspace = true chrono.workspace = true async-trait.workspace = true futures.workspace = true + +[dev-dependencies] +zeron-visual.workspace = true diff --git a/crates/mcp/src/jsonrpc.rs b/crates/mcp/src/jsonrpc.rs index 86cdd751ec..01b0c9e6c6 100644 --- a/crates/mcp/src/jsonrpc.rs +++ b/crates/mcp/src/jsonrpc.rs @@ -43,7 +43,14 @@ Discover hosts with `list_devices`, then `list_projects {device}`, \ `list_harnesses {device}` and `list_models {device, harness}`. A project fixes \ its host; with both project and device it must belong to that device. Use listed \ ids when names/paths repeat. Without project or device, creation uses the local engine. \ -If a chat is `awaitingInput`, answer it with `respond_to_input`."; +If a chat is `awaitingInput`, answer it with `respond_to_input`.\n\ +\n\ +When a chart, table, diagram, dashboard, or 3D view would say more than prose and \ +`show_visual` is available, write a Luau visual script and show it with `show_visual` before \ +your final reply. Zeron renders it with its own native interface and enforces its style: \ +content sits directly on the chat with no cards or boxed panels, text uses the interface font \ +and type scale, and colors are theme tokens that work on frosted glass or opaque, light or dark. \ +The reader sees the visual above that reply, so don't announce or restate it."; const PARSE_ERROR: i64 = -32700; const INVALID_REQUEST: i64 = -32600; diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs index c6d7490c88..7e7dc89bc0 100644 --- a/crates/mcp/src/tools.rs +++ b/crates/mcp/src/tools.rs @@ -225,11 +225,126 @@ fn catalog() -> Vec { }), }); } + tools.push(ToolDef { + name: SHOW_VISUAL, + description: SHOW_VISUAL_DESCRIPTION, + input_schema: json!({ + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "The Luau visual script: setup at the top level, ending with `return function() return ui.column { ... } end`." + }, + "title": { "type": "string", "description": "Short name for the visual." } + }, + "required": ["script", "title"] + }), + }); tools } +/// Shows a page in the CALLING chat, so it is offered only to an agent that +/// Zeron launched (`ZERON_CHAT_ID` set). +const SHOW_VISUAL: &str = "show_visual"; + +const SHOW_VISUAL_DESCRIPTION: &str = "\ +Show an interactive visual (chart, table, dashboard, diagram, simulation, 3D view) inline in this \ +chat, above your final text reply; call it before writing that reply. Use it when a visual says more \ +than prose. The reader already sees it, so the reply should not announce, locate, or restate it: add \ +only what the visual doesn't say.\n\ +\n\ +A visual is a Luau script (Lua 5.1 plus `+=`, `continue`, and `if` expressions). Zeron runs it in a \ +sandbox (no I/O or network; math.random is seeded, so data is the same on every render) and renders \ +what it returns with Zeron's own native interface. Build data and state at the top level, then return \ +a view function. Zeron calls the view again after every handler, so handlers just assign to state:\n\ +\n\ +local s = ui.state { metric = \"p95\" }\n\ +local data = { { region = \"us-east\", p95 = 182, p99 = 240 }, { region = \"eu-west\", p95 = 140, p99 = 210 } }\n\ +return function()\n\ + return ui.column { gap = 16,\n\ + ui.text { \"Latency by region\", size = \"lg\", weight = 600 },\n\ + ui.tabs { options = { \"p95\", \"p99\" }, value = s.metric, on_change = function(v) s.metric = v end },\n\ + ui.chart { type = \"bar\", height = 220, unit = \"ms\",\n\ + x = ui.map(data, function(d) return d.region end),\n\ + series = { { name = s.metric, values = ui.map(data, function(d) return d[s.metric] end) } } },\n\ + }\n\ +end\n\ +\n\ +Zeron runs the script before showing it (setup, render, every draw, and each tab, toggle, slider, and \ +button once) and returns the first error with its line; fix it and call again.\n\ +\n\ +Nodes are props tables: children are the array part, strings are plain text, nil and false children \ +are skipped, and lists of nodes flatten (so ui.map results can be children). Unknown props are errors.\n\ +- ui.column { gap = 8, align = \"start|center|end\", ... }, ui.row { gap = 12, align, justify = \ +\"start|center|end|between\", wrap = true, ... }, ui.grid { columns = 3, gap = 16, ... }. Any node \ +takes width = px or grow = true inside a row (sliders, charts, tables, canvases, and scenes fill a \ +row's spare width by default), and key = \"name\" to keep its hover or camera when the tree changes \ +shape. A row or column with on_click = function() end is clickable; hover = true gives \ +it the hover wash.\n\ +- ui.text { \"plain \", ui.span { \"bold\", weight = 600, color = \"accent\", mono = true }, size = \ +\"xs|sm|body|lg|xl|2xl\", weight = 400-700, color, mono, italic, align, truncate }, or ui.text \"string\".\n\ +- ui.stat { value = \"182 ms\", label = \"p95 latency\", delta = \"-12%\", tone = \"success\" }: a headline \ +figure over a muted label; tone colors the delta.\n\ +- ui.table { columns = { \"Region\", { label = \"p95\", align = \"end\" } }, rows = { { \"us-east\", 182 }, \ +... } }: laid out like a markdown table (bold header, hairline row rules), numbers right-aligned; cells \ +are strings, numbers, or nodes.\n\ +- ui.chart { type = \"bar|line|area|scatter\", height = 220, x = { labels }, series = { { name, values = \ +{ ... }, color } }, stacked, horizontal (bar), min, max, unit = \"ms\", prefix = \"$\", decimals, legend }. \ +Scatter series take points = { { x, y }, ... } (x_unit, x_decimals). Axes, gridlines, the legend, and a \ +hover readout are built in.\n\ +- ui.rule {} is a hairline divider; ui.space { size = 16 }; ui.link { label, url = \"https://...\" } \ +opens like a chat link.\n\ +- Controls: ui.tabs { options = { ... }, value, on_change = function(option) end }, ui.toggle { label, \ +value = bool, on_change = function(on) end }, ui.slider { label, value, min, max, step, unit, on_change \ += function(n) end }, ui.button { label, on_click = function() end, primary = bool }.\n\ +- ui.canvas { height = 240, draw = function(g, w, h, t) ... end, animate = bool, on_click = \ +function(x, y) end }: 2D drawing in pixels, origin top-left, w = the reply column's width. \ +g:line(x1, y1, x2, y2, { color, width, dash }), g:polyline(pts, { color, width }), g:polygon(pts, \ +{ fill, stroke, width }), g:rect(x, y, w, h, { fill, stroke, radius }), g:circle(x, y, r, { fill, \ +stroke }), g:arc(x, y, r, a0, a1, { color } or { fill, inner = radius } for pie and donut sectors; \ +radians clockwise from 3 o'clock), g:text(x, y, str, { size, color, weight, align = \ +\"start|center|end\", baseline = \"top|middle|bottom\", mono }). pts are { x1, y1, x2, y2, ... } or \ +{ { x, y }, ... }. g.pointer is { x, y, down } while the pointer is over the canvas (draw hover \ +effects with it); t is seconds, and animate = true redraws every frame while visible.\n\ +- ui.scene { height = 320, draw = function(s, t) ... end, orbit = true, spin = 0.3 (turntable \ +radians per second), camera = { yaw = 0.6, pitch = 0.45, zoom = 1 }, animate = bool }: real 3D, y up. \ +Zeron frames the camera on your geometry, the reader drags to orbit, and faces are lit and \ +depth-sorted. s:box { at = { x, y, z }, size = { w, h, d } or n, color }, s:sphere { at, radius, color }, \ +s:cylinder { at (base center), radius, height, color }, s:surface { x = { min, max }, z = { min, max }, \ +f = function(x, z) return y end or values = { { ... } }, detail = 32, colors = { \"chart1\", \"chart3\" } \ +(a ramp by height), wire = bool }, s:mesh { vertices = { { x, y, z }, ... }, faces = { { 1, 2, 3 }, ... }, \ +color }, s:line(a, b, { color, width }), s:path(points, { color, width }), s:point(at, { radius, \ +color, hint = \"shown on hover\" }), s:label(at, text, { color, size }), s:grid { size = 10, step = 1 }, \ +s:axes { size = 1, labels = { \"x\", \"y\", \"z\" } }.\n\ +- Helpers: ui.scale(d0, d1, r0, r1) returns a mapping function; ui.ticks(lo, hi, n); ui.fmt(n, { unit, \ +prefix, decimals }) groups digits; ui.map(list, function(v, i) end); ui.range(a, b, step); ui.clamp; \ +ui.lerp; ui.series(i) is \"chart\"; ui.color(token, alpha); ui.mix(a, b, t). print(...) output comes \ +back to you.\n\ +\n\ +Colors are theme tokens, never hex or color names: text; muted (labels, axes, captions); faint \ +(tertiary notes); accent (the one thing to notice); on-accent (text on an accent mark); success, \ +warning, danger (status only); border (hairlines, gridlines); chart1, chart2, chart3, chart4, \ +chart5, chart6 (series, in order; \ +chart1 is the accent). They follow the reader's theme, light or dark, on frosted glass or an opaque \ +surface, so never assume a background color.\n\ +\n\ +Style, enforced: layout nodes have no fill, border, shadow, padding, or font props. A visual is part \ +of the reply, not a widget, so lay content directly on the chat with type, spacing, and alignment: no \ +cards, tiles, boxed panels, banners, or canvas backgrounds. Stats are a row of plain ui.stat figures; \ +a chart is a plain heading, then the chart; separate sections with gap or one ui.rule. Text renders \ +in Zeron's interface font at the reader's text size: body for most text, sm and xs for labels and \ +notes, lg and xl for headings, 2xl for a headline figure. The width is the reply column (about 720px, \ +as narrow as 360px), so lay out fluidly with grow, wrap, and grid, and give charts, canvases, and \ +scenes heights of 160-420px."; + // ---- argument shapes --------------------------------------------------------- +#[derive(Deserialize)] +struct ShowVisualArgs { + script: String, + title: String, +} + #[derive(Deserialize)] struct BatchArgs { requests: Vec, @@ -436,11 +551,15 @@ impl Tools { } pub fn list(&self) -> Vec { + let in_chat = self.zeron.origin().chat_id.is_some(); catalog() + .into_iter() + .filter(|tool| in_chat || tool.name != SHOW_VISUAL) + .collect() } pub fn has(&self, name: &str) -> bool { - catalog().iter().any(|t| t.name == name) + self.list().iter().any(|t| t.name == name) } /// `Ok` is the tool's structured result; `Err` is a message the model @@ -463,6 +582,7 @@ impl Tools { "interrupt_chat" => self.interrupt_chat(parse(args)?).await, "respond_to_input" => self.respond_to_input(parse(args)?).await, "archive_chat" => self.archive_chat(parse(args)?).await, + SHOW_VISUAL => self.show_visual(parse(args)?).await, other => return Err(format!("unknown tool: {other}")), }; result.map_err(|e| format!("{e:#}")) @@ -974,6 +1094,30 @@ impl Tools { Ok(json!({ "chatId": chat.id, "title": chat.title, "archived": archived })) } + async fn show_visual(&self, args: ShowVisualArgs) -> anyhow::Result { + let chat_id = self + .zeron + .origin() + .chat_id + .clone() + .context("show_visual only works from an agent Zeron launched in a chat")?; + let shown = self + .zeron + .call( + zeron_rpc::methods::PUBLISH_VISUAL, + json!({ + "chatId": chat_id, + "title": args.title, + "script": args.script, + }), + ) + .await?; + Ok(json!({ + "visual": shown, + "message": "Shown to the reader above your reply. Don't mention or describe the visual; reply with only what it doesn't already say.", + })) + } + async fn interrupt_chat(&self, args: ChatArgs) -> anyhow::Result { let chat = self.zeron.resolve_chat(&args.chat).await?; let command_id = self @@ -1360,6 +1504,13 @@ mod tests { "status": "complete", "parts": [{ "kind": "text", "id": "t", "text": "hello back" }] } ]})), + methods::PUBLISH_VISUAL => { + self.writes + .lock() + .unwrap() + .push((method.to_owned(), params.clone())); + RpcReply::Value(json!({ "id": "v:1", "title": params["title"] })) + } methods::MUTATE | methods::QUEUE_COMMAND | methods::QUEUE_MESSAGE => { self.writes .lock() @@ -1397,6 +1548,59 @@ mod tests { } } + #[test] + fn the_show_visual_example_passes_its_own_check() { + let start = SHOW_VISUAL_DESCRIPTION.find("local s = ui.state").unwrap(); + let end = start + + SHOW_VISUAL_DESCRIPTION[start..] + .find("\n\nZeron runs the script") + .unwrap(); + let report = zeron_visual::check(&SHOW_VISUAL_DESCRIPTION[start..end], 1).unwrap(); + assert_eq!(report.interactions, 2, "both tabs"); + } + + #[tokio::test] + async fn show_visual_is_offered_only_inside_a_chat() { + let world = Arc::new(World::default()); + let outside = tools(world.clone(), Origin::default()); + assert!(!outside.has("show_visual")); + assert!( + outside + .call( + "show_visual", + json!({ "script": "return ui.text 'x'", "title": "x" }) + ) + .await + .is_err() + ); + + let inside = tools( + world.clone(), + Origin { + chat_id: Some("chat-alpha-1".into()), + device_id: Some("dev-local".into()), + }, + ); + assert!(inside.has("show_visual")); + // The guidance names every color the runtime accepts. + for name in zeron_visual::TOKEN_NAMES { + assert!(SHOW_VISUAL_DESCRIPTION.contains(name), "{name}"); + } + let shown = inside + .call( + "show_visual", + json!({ "script": "return ui.text 'chart'", "title": "Chart" }), + ) + .await + .unwrap(); + assert_eq!(shown["visual"]["title"], "Chart"); + let writes = world.writes.lock().unwrap(); + let (method, params) = writes.last().unwrap(); + assert_eq!(method, methods::PUBLISH_VISUAL); + assert_eq!(params["chatId"], "chat-alpha-1"); + assert_eq!(params["script"], "return ui.text 'chart'"); + } + #[tokio::test] async fn list_and_read_chats() { let world = Arc::new(World::default()); diff --git a/crates/mcp/src/transcript.rs b/crates/mcp/src/transcript.rs index 4ad48980a3..58f8c0025d 100644 --- a/crates/mcp/src/transcript.rs +++ b/crates/mcp/src/transcript.rs @@ -116,6 +116,7 @@ fn render_one(entry: &SessionMessageEntry, options: RenderOptions) -> RenderedMe } } MessagePart::Image { name, .. } => tools.push(format!("image: {name}")), + MessagePart::Visual { title, .. } => tools.push(format!("visual: {title}")), MessagePart::Reasoning { text, .. } => { if options.include_reasoning && !text.trim().is_empty() { reasoning_parts.push(text.as_str()); diff --git a/crates/proto/src/agent.rs b/crates/proto/src/agent.rs index d7973c0355..d74c330528 100644 --- a/crates/proto/src/agent.rs +++ b/crates/proto/src/agent.rs @@ -415,6 +415,11 @@ pub const SUBAGENT_INPUT_KEEP: [&str; 5] = [ "subagent_type", ]; +/// Bounds on an agent's inline visual ([`AgentEvent::Visual`]): a Luau +/// script Zeron runs in a sandbox and renders natively. +pub const VISUAL_MAX_BYTES: usize = 256 * 1024; +pub const VISUAL_MAX_TITLE_CHARS: usize = 200; + /// Where one checklist item stands. Claude's TodoWrite, OpenCode and ACP plans /// distinguish the item being worked on from those still waiting; Codex and /// Cursor only report done / not done. @@ -571,6 +576,15 @@ pub enum AgentEvent { name: String, mime_type: String, }, + /// An agent-authored visual shown inline in the reply: a Luau script + /// that builds native UI (`zeron-visual`). The engine emits it when the + /// agent calls Zeron's `show_visual` MCP tool; `path` is the stored + /// script on the host device. + Visual { + id: String, + path: String, + title: String, + }, ReasoningDelta { text: String, }, diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index c9c219b8c9..6cef2e1da4 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -229,6 +229,9 @@ pub mod methods { pub const UPLOAD_CHUNK: &str = "UploadChunk"; pub const UPLOAD_COMMIT: &str = "UploadCommit"; pub const READ_ATTACHMENT_CHUNK: &str = "ReadAttachmentChunk"; + /// Store an agent's HTML page and show it inline in that chat's live + /// turn (`show_visual`). Local only: the calling agent runs on this device. + pub const PUBLISH_VISUAL: &str = "PublishVisual"; /// Lazy full-tool-output fetch from the R2 sidecar by doc-resident ref /// (chat2-sync A3). Edge-direct from any device — never relay-forwarded. pub const FETCH_TOOL_BLOB: &str = "FetchToolBlob"; diff --git a/crates/ui/Cargo.toml b/crates/ui/Cargo.toml index 40fb4a0da5..44ba704ae4 100644 --- a/crates/ui/Cargo.toml +++ b/crates/ui/Cargo.toml @@ -74,6 +74,7 @@ zeron-rpc.workspace = true zeron-update.workspace = true zeron-syntax.workspace = true zeron-theme.workspace = true +zeron-visual.workspace = true gpui.workspace = true gpui_platform.workspace = true gpui_tokio.workspace = true diff --git a/crates/ui/src/lib.rs b/crates/ui/src/lib.rs index 91a451c7c7..b9d1eca902 100644 --- a/crates/ui/src/lib.rs +++ b/crates/ui/src/lib.rs @@ -79,6 +79,7 @@ pub mod theme_library; pub mod toast; pub mod transcript; pub mod typography; +pub mod visuals; pub mod voice; mod workspace_links; diff --git a/crates/ui/src/transcript.rs b/crates/ui/src/transcript.rs index 20545b8cfa..515d6c65c3 100644 --- a/crates/ui/src/transcript.rs +++ b/crates/ui/src/transcript.rs @@ -1025,6 +1025,12 @@ pub enum RowKind { name: String, mime_type: String, }, + /// An agent's `show_visual` script, rendered inline by [`crate::visuals`]. + Visual { + owner: String, + path: String, + title: String, + }, User { /// Visible prompt (attachment-ref trailer already stripped). When the /// prompt carries file mentions this is the *projected* display text — @@ -1625,6 +1631,28 @@ pub fn rows_for_entry( }, }); } + MessagePart::Visual { + id: part_id, + path, + title, + } => { + rows.push(Row { + id: format!("{}#{}", entry.id, part_id).into(), + version: fnv1a( + format!("{}\0{path}\0{title}", entry.device_id).as_bytes(), + ), + turn_start: false, + entry_id: entry_id.clone(), + timestamp: None, + copy_text: None, + compact_fold: None, + kind: RowKind::Visual { + owner: entry.device_id.clone(), + path: path.clone(), + title: title.clone(), + }, + }); + } MessagePart::Input { id: part_id, questions, @@ -3339,6 +3367,8 @@ pub struct Transcript { /// In-flight ReadAttachmentChunk loads, keyed by device, path and validation /// policy; results land in the global attachment cache. attachment_loads: HashMap>, + /// Inline `show_visual` visuals, keyed by row id. + visuals: HashMap>, /// Scheduled retry wake-ups for errored sources (the 2s→15s ladder). attachment_retries: HashMap<(String, String), Task<()>>, /// Sidecar blob fetches keyed by doc ref (`chatId/partId[.diff]`, @@ -3634,6 +3664,7 @@ impl Transcript { image_menu_focus: cx.focus_handle(), image_copy: None, attachment_loads: HashMap::new(), + visuals: HashMap::new(), attachment_retries: HashMap::new(), blob_details: HashMap::new(), blob_fetch_order: HashMap::new(), @@ -6677,6 +6708,51 @@ impl Transcript { } } + fn render_visual( + &mut self, + row_id: &SharedString, + owner: &str, + path: &str, + title: &str, + cx: &mut Context, + ) -> AnyElement { + let link_ui = self.link_ui(cx); + let frame = match self.visuals.get(row_id) { + Some(frame) => frame.clone(), + None => { + let state = self.state.read(cx); + let engine = state.engine().cloned(); + // Relay-forward only for a genuinely remote owner. + let target = + (state.local_device_id.as_deref() != Some(owner)).then(|| owner.to_owned()); + let frame = cx.new(|cx| { + crate::visuals::VisualFrame::new( + row_id.clone(), + title.to_owned(), + path.to_owned(), + engine, + target, + cx, + ) + }); + // A visual whose tree changes shape (a tab switch, a load) + // resizes its row, like a diagram finishing. + let id = row_id.clone(); + cx.subscribe( + &frame, + move |this, _, _: &crate::visuals::VisualLayoutChanged, cx| { + this.diagram_layout_changed(std::slice::from_ref(&id), cx); + }, + ) + .detach(); + self.visuals.insert(row_id.clone(), frame.clone()); + frame + } + }; + frame.update(cx, |frame, _| frame.set_link_ui(link_ui)); + div().w_full().child(frame).into_any_element() + } + /// The right-aligned thumbnail strip above a user bubble. fn render_user_attachments( &mut self, @@ -7451,6 +7527,9 @@ impl Transcript { name, mime_type, } => self.render_generated_image(&row.id, owner, path, name, mime_type, cx), + RowKind::Visual { owner, path, title } => { + self.render_visual(&row.id, owner, path, title, cx) + } RowKind::ErrorChip { message } => error_chip(message.clone(), &theme), RowKind::ForkMarker { source_title, .. } => fork_marker(source_title.clone(), &theme), }; @@ -10004,6 +10083,12 @@ fn entry_fingerprint(entry: &SessionMessageEntry, pending: bool) -> u64 { acc.push(0); } } + if let MessagePart::Visual { path, title, .. } = part { + for field in [path, title] { + acc.extend_from_slice(field.as_bytes()); + acc.push(0); + } + } if let MessagePart::Input { resolved, .. } = part { acc.push(0x10 | *resolved as u8); } diff --git a/crates/ui/src/visuals.rs b/crates/ui/src/visuals.rs new file mode 100644 index 0000000000..b7b28c666d --- /dev/null +++ b/crates/ui/src/visuals.rs @@ -0,0 +1,2282 @@ +//! Agent visuals: Luau scripts an agent shows inline in its reply through +//! Zeron's `show_visual` MCP tool (charts, tables, dashboards, simulations, +//! 3D views). +//! +//! A script returns a tree of `ui.*` nodes ([`zeron_visual`]), and this module +//! renders it with the same GPUI elements, type scale, and theme as the +//! transcript around it. Nothing foreign is hosted: a visual scrolls, clips, +//! fades under the glass composer, and takes the pointer exactly like a +//! markdown table, on every platform. Controls call the script's handlers, +//! which change its state, and the view renders again. Canvases and scenes +//! run their draw functions at paint time and are painted as GPUI paths and +//! shaped text in theme colors. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::Arc; +use std::time::Instant; + +use gpui::AppContext as _; +use gpui::prelude::FluentBuilder as _; +use gpui::{ + AnyElement, App, Bounds, Context, EventEmitter, FontStyle, FontWeight, Hsla, + InteractiveElement, IntoElement, MouseButton, MouseMoveEvent, ParentElement, PathBuilder, + Pixels, Point, Render, SharedString, StatefulInteractiveElement, Styled, StyledText, Task, + TextAlign, TextRun, Window, canvas, div, point, px, +}; +use zeron_visual::{ + Align, ChartKind, DrawCmd, HandlerArg, HandlerId, Justify, Node, NodeKind, Prim, Token, +}; + +use crate::markdown::render::LinkUi; +use crate::state::EngineHandle; +use crate::theme::Theme; + +/// The visual's tree changed shape; the transcript remeasures its row. +pub struct VisualLayoutChanged; + +enum Load { + Loading, + Ready, + Failed(SharedString), +} + +/// One inline visual in a transcript. +pub struct VisualFrame { + id: SharedString, + title: SharedString, + load: Load, + runtime: Option>>, + tree: Option>, + /// Bumped on every render of the tree: draw caches key on it. + generation: u64, + /// The latest handler or draw failure; the last good tree stays up. + error: Option, + link_ui: Option, + local: Rc>, + started: Instant, + _tasks: Vec>, +} + +impl EventEmitter for VisualFrame {} + +/// View-local state the script doesn't own, keyed by node key or tree path. +#[derive(Default)] +struct Local { + /// The pointer over a canvas, chart, or scene, in window coordinates. + pointers: HashMap>, + pressed: Option, + /// Where each chart's plot was painted, for hover hit-testing. + charts: HashMap, + cameras: HashMap, + canvases: HashMap, + scenes: HashMap, + draw_error: Option, +} + +struct CanvasCache { + generation: u64, + size: (f32, f32), + pointer: Option<(f32, f32, bool)>, + commands: Rc>, +} + +struct SceneCache { + generation: u64, + scene: Rc, +} + +#[derive(Clone, Copy)] +struct Orbit { + yaw: f32, + pitch: f32, + /// Where the turntable started turning from `yaw`. + spin_epoch: Instant, + drag: Option<(Point, f32, f32)>, +} + +#[derive(Clone, Copy, Default)] +struct ChartGeometry { + plot: Bounds, +} + +/// The drag payload for scene orbits and sliders: which node is dragging. +#[derive(Clone)] +struct VisualDrag(SharedString); + +struct DragGhost; + +impl Render for DragGhost { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + gpui::Empty + } +} + +impl VisualFrame { + pub fn new( + id: SharedString, + title: String, + path: String, + engine: Option, + target_device_id: Option, + cx: &mut Context, + ) -> Self { + let mut tasks = Vec::new(); + let load = match engine { + Some(engine) => { + tasks.push(cx.spawn(async move |this, cx| { + let script = crate::attachments::read_attachment_bytes( + &engine, + cx.background_executor(), + target_device_id.as_deref(), + &path, + None, + ) + .await + .and_then(|(_, _, bytes)| String::from_utf8(bytes).ok()); + let _ = this.update(cx, |this, cx| match script { + Some(script) => this.start(&script, cx), + None => { + this.load = Load::Failed("Visual unavailable".into()); + cx.notify(); + } + }); + })); + Load::Loading + } + None => Load::Failed("Visual unavailable".into()), + }; + Self { + id, + title: title.into(), + load, + runtime: None, + tree: None, + generation: 0, + error: None, + link_ui: None, + local: Rc::default(), + started: Instant::now(), + _tasks: tasks, + } + } + + pub fn set_link_ui(&mut self, link_ui: Option) { + self.link_ui = link_ui; + } + + /// Run a script's setup and first render (the engine already ran the + /// same script before publishing it, so failures here are rare). + fn start(&mut self, script: &str, cx: &mut Context) { + // The row id ends with the part's visual id, which the engine + // seeded its check with: the reader sees the same random data. + let visual_id = self.id.rsplit('#').next().unwrap_or(&self.id); + let seed = zeron_visual::seed_for(visual_id); + match zeron_visual::Visual::load(script, seed) { + Ok(visual) => { + self.runtime = Some(Rc::new(RefCell::new(visual))); + self.load = Load::Ready; + self.rerender(cx); + } + Err(error) => { + tracing::warn!(visual = %self.id, %error, "visual script failed to load"); + self.load = Load::Failed(format!("Visual failed: {error}").into()); + cx.notify(); + } + } + } + + fn rerender(&mut self, cx: &mut Context) { + let Some(runtime) = self.runtime.clone() else { + return; + }; + let rendered = runtime.borrow_mut().render(); + match rendered { + Ok(tree) => { + let changed = self.tree.as_deref() != Some(&tree); + self.tree = Some(Rc::new(tree)); + self.generation += 1; + self.error = None; + if changed { + cx.emit(VisualLayoutChanged); + } + } + Err(error) => self.fail(error, cx), + } + cx.notify(); + } + + fn fail(&mut self, error: zeron_visual::VisualError, cx: &mut Context) { + tracing::warn!(visual = %self.id, %error, "visual script error"); + self.error = Some(error.message.into()); + cx.emit(VisualLayoutChanged); + cx.notify(); + } + + fn dispatch(&mut self, handler: HandlerId, arg: HandlerArg, cx: &mut Context) { + let Some(runtime) = self.runtime.clone() else { + return; + }; + let called = runtime.borrow_mut().call(handler, arg); + match called { + Ok(()) => self.rerender(cx), + Err(error) => self.fail(error, cx), + } + } + + fn open_link(&self, url: &str, window: &mut Window, cx: &mut App) { + crate::markdown::render::activate_link( + crate::markdown::links::LinkTarget::new(url, url), + crate::markdown::links::LinkAction::Primary, + self.link_ui.as_ref(), + window, + cx, + ); + } + + fn element_id(&self, path: &str) -> SharedString { + format!("{}:{path}", self.id).into() + } + + fn time(&self) -> f64 { + self.started.elapsed().as_secs_f64() + } +} + +// ---- colors and type ------------------------------------------------------ + +/// Categorical chart colors: the theme's accent, then the five fixed hues +/// that sit farthest from it, so no two series read alike whatever the +/// accent. Solid, deeper on light and brighter on dark, so they hold on +/// glass and opaque surfaces alike. +fn chart_series(theme: &Theme) -> [Hsla; 6] { + // (hue in turns, light tone, dark tone): teal, amber, rose, violet, lime, sky. + const FIXED: [(f32, u32, u32); 6] = [ + (0.486, 0x0d9488, 0x2dd4bf), + (0.106, 0xd97706, 0xfbbf24), + (0.964, 0xe11d48, 0xfb7185), + (0.728, 0x7c3aed, 0xa78bfa), + (0.236, 0x65a30d, 0xa3e635), + (0.553, 0x0284c7, 0x38bdf8), + ]; + let accent = theme.accent_strong; + let distance = |hue: f32| { + let d = (hue - accent.h).rem_euclid(1.0); + d.min(1.0 - d) + }; + // A grey accent has no hue to collide with; drop the last fixed color. + let closest = if accent.s < 0.15 { + FIXED.len() - 1 + } else { + (0..FIXED.len()) + .min_by(|a, b| distance(FIXED[*a].0).total_cmp(&distance(FIXED[*b].0))) + .unwrap_or(0) + }; + let light = theme.appearance.is_light(); + let mut out = [accent; 6]; + for (slot, (_, (_, l, d))) in FIXED + .iter() + .enumerate() + .filter(|(ix, _)| *ix != closest) + .enumerate() + { + out[slot + 1] = gpui::rgb(if light { *l } else { *d }).into(); + } + out +} + +/// Theme colors for one paint, resolved once. +struct Palette { + text: Hsla, + muted: Hsla, + faint: Hsla, + accent: Hsla, + on_accent: Hsla, + success: Hsla, + warning: Hsla, + danger: Hsla, + border: Hsla, + chart: [Hsla; 6], +} + +impl Palette { + fn new(theme: &Theme) -> Self { + Self { + text: theme.text, + muted: theme.text_muted, + faint: theme.text_faint, + accent: theme.accent, + on_accent: theme.on_accent, + success: theme.success, + warning: theme.warning, + danger: theme.danger, + border: theme.border, + chart: chart_series(theme), + } + } + + fn token(&self, token: Token) -> Hsla { + match token { + Token::Text => self.text, + Token::Muted => self.muted, + Token::Faint => self.faint, + Token::Accent => self.accent, + Token::OnAccent => self.on_accent, + Token::Success => self.success, + Token::Warning => self.warning, + Token::Danger => self.danger, + Token::Border => self.border, + Token::Chart(i) => self.chart[usize::from(i) % 6], + } + } + + fn color(&self, color: zeron_visual::Color) -> Hsla { + let mut out = self.token(color.token); + if let Some((other, t)) = color.mix { + out = mix_rgb(out, self.token(other), t); + } + out.a *= color.alpha; + out + } +} + +/// Blend in RGB, so a ramp between two hues doesn't swing through a third. +fn mix_rgb(a: Hsla, b: Hsla, t: f32) -> Hsla { + let (a, b) = (a.to_rgb(), b.to_rgb()); + let lerp = |x: f32, y: f32| x + (y - x) * t; + gpui::Rgba { + r: lerp(a.r, b.r), + g: lerp(a.g, b.g), + b: lerp(a.b, b.b), + a: lerp(a.a, b.a), + } + .into() +} + +/// A face's color under the scene light. +fn shaded(color: Hsla, shade: f32) -> Hsla { + Hsla { + l: (color.l * (0.42 + 0.62 * shade)).clamp(0.0, 0.97), + ..color + } +} + +fn weight(weight: zeron_visual::Weight) -> FontWeight { + match weight { + zeron_visual::Weight::Regular => FontWeight::NORMAL, + zeron_visual::Weight::Medium => FontWeight::MEDIUM, + zeron_visual::Weight::Semibold => FontWeight::SEMIBOLD, + zeron_visual::Weight::Bold => FontWeight::BOLD, + } +} + +/// Visuals set figures in tabular numerals, so columns of numbers align. +fn ui_font(theme: &Theme, mono: bool, w: FontWeight, italic: bool) -> gpui::Font { + let mut font = gpui::font(if mono { + theme.font_mono.clone() + } else { + theme.font_sans.clone() + }); + font.weight = w; + if italic { + font.style = FontStyle::Italic; + } + font.features = gpui::FontFeatures(Arc::new(vec![("tnum".into(), 1)])); + font +} + +fn rems(size: zeron_visual::TextSize) -> (gpui::Rems, gpui::Rems) { + let (size, line_height) = size.metrics(); + ( + crate::typography::ui_rems(size), + crate::typography::ui_rems(line_height), + ) +} + +/// A text size in window pixels at the reader's interface size. +fn text_px(size: zeron_visual::TextSize, window: &Window) -> (Pixels, Pixels) { + let (size, line_height) = rems(size); + ( + size.to_pixels(window.rem_size()), + line_height.to_pixels(window.rem_size()), + ) +} + +// ---- rendering ------------------------------------------------------------ + +impl Render for VisualFrame { + fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + let theme = Theme::of(cx).clone(); + let body = match (&self.load, self.tree.clone()) { + (Load::Failed(message), _) => status_line(message.clone(), theme.text_muted), + (Load::Ready, Some(tree)) => { + let palette = Palette::new(&theme); + let root = self.id.to_string(); + self.node(&tree, &root, &theme, &palette, window, cx) + } + _ => status_line("Loading visual…".into(), theme.text_faint), + }; + let error = self + .error + .clone() + .or_else(|| self.local.borrow().draw_error.clone().map(Into::into)); + div() + .id(self.element_id("frame")) + .w_full() + .flex() + .flex_col() + .gap(px(8.0)) + .role(gpui::Role::Group) + .aria_label(self.title.clone()) + .child(body) + .when_some(error, |el, error| { + el.child(status_line( + format!("Visual error: {error}").into(), + theme.danger, + )) + }) + } +} + +fn status_line(message: SharedString, color: Hsla) -> AnyElement { + div() + .text_size(crate::typography::ui_rems(12.0)) + .line_height(crate::typography::ui_rems(18.0)) + .text_color(color) + .child(message) + .into_any_element() +} + +impl VisualFrame { + fn key(node: &Node, path: &str) -> SharedString { + match &node.key { + Some(key) => format!("key:{key}").into(), + None => path.to_owned().into(), + } + } + + fn node( + &mut self, + node: &Node, + path: &str, + theme: &Theme, + palette: &Palette, + window: &mut Window, + cx: &mut Context, + ) -> AnyElement { + let key = Self::key(node, path); + match &node.kind { + NodeKind::Stack(stack) => self.stack(stack, path, theme, palette, window, cx), + NodeKind::Grid { + columns, + gap, + children, + } => { + let mut grid = div().w_full().grid().grid_cols(*columns).gap(px(*gap)); + for (ix, child) in children.iter().enumerate() { + let child_path = format!("{path}.{ix}"); + grid = grid.child(div().min_w_0().child(self.node( + child, + &child_path, + theme, + palette, + window, + cx, + ))); + } + grid.into_any_element() + } + NodeKind::Text(text) => text_element(text, theme, palette), + NodeKind::Rule => div() + .h(px(1.0)) + .w_full() + .bg(theme.border) + .into_any_element(), + NodeKind::Space(size) => div().flex_none().size(px(*size)).into_any_element(), + NodeKind::Stat { + value, + label, + delta, + tone, + } => stat(value, label, delta.as_deref(), *tone, theme, palette), + NodeKind::Table(table) => self.table(table, path, theme, palette, window, cx), + NodeKind::Chart(chart) => self.chart(chart, key, theme, palette, cx), + NodeKind::Canvas(canvas) => self.canvas(canvas, key, cx), + NodeKind::Scene(scene) => self.scene(scene, key, cx), + NodeKind::Button { + label, + on_click, + primary, + } => { + let handler = *on_click; + crate::settings::widgets::action_button( + theme, + if *primary { + crate::settings::widgets::ActionTone::Solid + } else { + crate::settings::widgets::ActionTone::Filled + }, + ) + .flex_none() + .id(self.element_id(path)) + .role(gpui::Role::Button) + .aria_label(SharedString::from(label.clone())) + .child(label.clone()) + .on_click(cx.listener(move |this, _, _, cx| { + cx.stop_propagation(); + this.dispatch(handler, HandlerArg::None, cx); + })) + .into_any_element() + } + NodeKind::Tabs { + options, + value, + on_change, + } => { + let reduced = cx.reduce_motion(); + let mut row = div().flex().flex_row().flex_wrap().gap(px(2.0)); + for (ix, option) in options.iter().enumerate() { + let selected = option == value; + let tab_key = format!("{}-tab-{ix}", self.element_id(path)); + let t = crate::settings::widgets::tab_selection_t( + window, + tab_key.clone(), + selected, + reduced, + ); + let (handler, option_value) = (*on_change, option.clone()); + row = row.child( + crate::settings::widgets::section_tab( + theme, + selected, + t, + SharedString::from(tab_key.clone()), + format!("{tab_key}-hover"), + ) + .cursor_pointer() + .role(gpui::Role::Tab) + .aria_label(SharedString::from(option.clone())) + .child(option.clone()) + .on_click(cx.listener(move |this, _, _, cx| { + cx.stop_propagation(); + this.dispatch(handler, HandlerArg::Text(option_value.clone()), cx); + })), + ); + } + row.into_any_element() + } + NodeKind::Toggle { + label, + value, + on_change, + } => { + let (handler, next) = (*on_change, !*value); + div() + .id(self.element_id(path)) + .flex() + .flex_row() + .items_center() + .gap(px(10.0)) + .cursor_pointer() + .role(gpui::Role::Switch) + .aria_label(SharedString::from(label.clone())) + .child(crate::settings::widgets::toggle_switch( + theme, + *value, + self.element_id(path), + )) + .child( + div() + .text_size(crate::typography::ui_rems(14.0)) + .line_height(crate::typography::ui_rems(22.0)) + .text_color(theme.text) + .child(label.clone()), + ) + .on_click(cx.listener(move |this, _, _, cx| { + cx.stop_propagation(); + this.dispatch(handler, HandlerArg::Bool(next), cx); + })) + .into_any_element() + } + NodeKind::Slider { + label, + value, + min, + max, + step, + on_change, + format, + } => self.slider( + SliderSpec { + label: label.clone(), + value: *value, + min: *min, + max: *max, + step: *step, + handler: *on_change, + shown: format.format(*value), + }, + key, + theme, + cx, + ), + NodeKind::Link { label, url } => { + let url = url.clone(); + div() + .id(self.element_id(path)) + .text_size(crate::typography::ui_rems(14.0)) + .line_height(crate::typography::ui_rems(22.0)) + .text_color(theme.accent) + .cursor_pointer() + .hover(|s| s.underline()) + .role(gpui::Role::Link) + .child(label.clone()) + .on_click(cx.listener(move |this, _, window, cx| { + cx.stop_propagation(); + this.open_link(&url, window, cx); + })) + .into_any_element() + } + } + } + + fn stack( + &mut self, + stack: &zeron_visual::Stack, + path: &str, + theme: &Theme, + palette: &Palette, + window: &mut Window, + cx: &mut Context, + ) -> AnyElement { + let horizontal = stack.axis == zeron_visual::Axis::Horizontal; + let mut el = div().flex().gap(px(stack.gap)); + el = if horizontal { + el.flex_row().when(stack.wrap, |el| el.flex_wrap()) + } else { + el.flex_col().w_full() + }; + el = match stack.align { + Align::Start if horizontal => el.items_start(), + Align::Start => el, + Align::Center => el.items_center(), + Align::End => el.items_end(), + }; + el = match stack.justify { + Justify::Start => el, + Justify::Center => el.justify_center(), + Justify::End => el.justify_end(), + Justify::Between => el.justify_between(), + }; + if horizontal { + el = el.w_full(); + } + for (ix, child) in stack.children.iter().enumerate() { + let child_path = format!("{path}.{ix}"); + let rendered = self.node(child, &child_path, theme, palette, window, cx); + // Nodes with no natural width (a slider's track, a plot) fill a + // row's spare width unless the script sized them. + let fluid = matches!( + child.kind, + NodeKind::Slider { .. } + | NodeKind::Chart(_) + | NodeKind::Canvas(_) + | NodeKind::Scene(_) + | NodeKind::Table(_) + | NodeKind::Rule + ); + let mut wrapper = div().min_w_0(); + wrapper = match (child.width, child.grow) { + (Some(width), _) => wrapper.flex_none().w(px(width)), + (None, true) => wrapper.flex_1(), + (None, false) if horizontal && fluid => wrapper.flex_1(), + (None, false) if horizontal => wrapper.flex_shrink(1.0), + (None, false) => wrapper, + }; + el = el.child(wrapper.child(rendered)); + } + if stack.on_click.is_none() && !stack.hover { + return el.into_any_element(); + } + let hover = theme.glass_hover(); + let mut el = el + .id(self.element_id(path)) + .rounded(px(6.0)) + .when(stack.hover || stack.on_click.is_some(), |el| { + el.hover(move |s| s.bg(hover)) + }); + if let Some(handler) = stack.on_click { + el = el + .cursor_pointer() + .role(gpui::Role::Button) + .on_click(cx.listener(move |this, _, _, cx| { + cx.stop_propagation(); + this.dispatch(handler, HandlerArg::None, cx); + })); + } + el.into_any_element() + } + + /// The markdown table's frameless treatment: a bold header and hairlines + /// between rows, no outer box or fills. + fn table( + &mut self, + table: &zeron_visual::TableNode, + path: &str, + theme: &Theme, + palette: &Palette, + window: &mut Window, + cx: &mut Context, + ) -> AnyElement { + use crate::markdown::render::{TABLE_CELL_PADDING, TABLE_HEADER_WEIGHT, table_hairline}; + let columns = table + .columns + .len() + .max(table.rows.iter().map(Vec::len).max().unwrap_or(0)); + let cell = |align: Align, width: Option| { + let cell = div() + .min_w_0() + .px(px(TABLE_CELL_PADDING)) + .py(px(8.0)) + .flex() + .flex_row(); + let cell = match width { + Some(width) => cell.flex_none().w(px(width)), + None => cell.flex_1().flex_basis(px(0.0)), + }; + match align { + Align::Start => cell, + Align::Center => cell.justify_center(), + Align::End => cell.justify_end(), + } + }; + let column = |c: usize| table.columns.get(c).cloned().unwrap_or_default(); + let mut el = div().flex().flex_col().w_full(); + if table.columns.iter().any(|c| !c.label.is_empty()) { + let mut header = div().flex().flex_row(); + for c in 0..columns { + let column = column(c); + header = header.child( + cell(column.align, column.width).child( + div() + .text_size(crate::typography::ui_rems(14.0)) + .line_height(crate::typography::ui_rems(22.0)) + .font_weight(TABLE_HEADER_WEIGHT) + .text_color(theme.text) + .child(column.label.clone()), + ), + ); + } + el = el.child(header); + } + let hover = theme.glass_hover(); + for (r, row) in table.rows.iter().enumerate() { + if r > 0 || !table.columns.is_empty() { + el = el.child(div().flex_none().h(px(1.0)).w_full().bg(table_hairline())); + } + let mut row_el = div() + .id(self.element_id(&format!("{path}.r{r}"))) + .flex() + .flex_row() + .when(table.hover, |el| el.hover(move |s| s.bg(hover))); + for c in 0..columns { + let column = column(c); + let content = match row.get(c) { + Some(node) => { + let child_path = format!("{path}.{r}.{c}"); + self.node(node, &child_path, theme, palette, window, cx) + } + None => gpui::Empty.into_any_element(), + }; + row_el = row_el.child(cell(column.align, column.width).child(content)); + } + el = el.child(row_el); + } + el.into_any_element() + } +} + +fn text_element(text: &zeron_visual::Text, theme: &Theme, palette: &Palette) -> AnyElement { + let (size, line_height) = rems(text.size); + let base_color = text.color.map(|c| palette.color(c)).unwrap_or(theme.text); + let mut content = String::new(); + let mut runs = Vec::new(); + for span in &text.spans { + if span.text.is_empty() { + continue; + } + content.push_str(&span.text); + runs.push(TextRun { + len: span.text.len(), + font: ui_font( + theme, + text.mono || span.mono, + weight(span.weight.unwrap_or(text.weight)), + text.italic || span.italic, + ), + color: span.color.map(|c| palette.color(c)).unwrap_or(base_color), + background_color: None, + underline: None, + strikethrough: None, + }); + } + let el = div() + .min_w_0() + .text_size(size) + .line_height(line_height) + .text_color(base_color) + .when(text.truncate, |el| el.truncate()); + let el = match text.align { + Align::Start => el, + Align::Center => el.text_center(), + Align::End => el.text_right(), + }; + el.child(StyledText::new(content).with_runs(runs)) + .into_any_element() +} + +/// A headline figure over a short label: plain type, no tile. +fn stat( + value: &str, + label: &str, + delta: Option<&str>, + tone: Option, + theme: &Theme, + palette: &Palette, +) -> AnyElement { + let (size, line_height) = rems(zeron_visual::TextSize::Xxl); + // A tone marks the change, not the figure; a figure with no change + // takes the tone itself. + let tone = tone.map(|c| palette.color(c)); + let has_delta = delta.is_some_and(|d| !d.is_empty()); + let value_color = tone.filter(|_| !has_delta).unwrap_or(theme.text); + div() + .flex() + .flex_col() + .gap(px(2.0)) + .child( + div() + .flex() + .flex_row() + .items_baseline() + .gap(px(6.0)) + .child( + div() + .text_size(size) + .line_height(line_height) + .text_color(value_color) + .child(StyledText::new(value.to_owned()).with_runs(vec![TextRun { + len: value.len(), + font: ui_font(theme, false, FontWeight::SEMIBOLD, false), + color: value_color, + background_color: None, + underline: None, + strikethrough: None, + }])), + ) + .when_some(delta.filter(|d| !d.is_empty()), |el, delta| { + el.child( + div() + .text_size(crate::typography::ui_rems(12.0)) + .line_height(crate::typography::ui_rems(18.0)) + .text_color(tone.unwrap_or(theme.text_muted)) + .child(delta.to_owned()), + ) + }), + ) + .when(!label.is_empty(), |el| { + el.child( + div() + .text_size(crate::typography::ui_rems(12.0)) + .line_height(crate::typography::ui_rems(18.0)) + .text_color(theme.text_muted) + .child(label.to_owned()), + ) + }) + .into_any_element() +} + +struct SliderSpec { + label: Option, + value: f64, + min: f64, + max: f64, + step: f64, + handler: HandlerId, + shown: String, +} + +impl SliderSpec { + /// The value under `x` on a track painted inset by half a thumb. + fn value_at(&self, x: Pixels, bounds: Bounds) -> f64 { + let inset = px(SLIDER_THUMB / 2.0); + let width = (bounds.size.width - inset * 2.0).max(px(1.0)); + let t = f64::from(((x - bounds.origin.x - inset) / width).clamp(0.0, 1.0)); + let mut value = self.min + (self.max - self.min) * t; + if self.step > 0.0 { + value = self.min + ((value - self.min) / self.step).round() * self.step; + } + value.clamp(self.min, self.max) + } +} + +const SLIDER_TRACK: f32 = 4.0; +const SLIDER_THUMB: f32 = 14.0; + +impl VisualFrame { + fn slider( + &mut self, + spec: SliderSpec, + key: SharedString, + theme: &Theme, + cx: &mut Context, + ) -> AnyElement { + let fraction = ((spec.value - spec.min) / (spec.max - spec.min)).clamp(0.0, 1.0) as f32; + let spec = Rc::new(spec); + let (track, fill, thumb) = (theme.border, theme.accent, theme.text); + let press = { + let (spec, key) = (spec.clone(), key.clone()); + cx.listener(move |this, event: &gpui::MouseDownEvent, window, cx| { + cx.stop_propagation(); + this.local.borrow_mut().pressed = Some(key.clone()); + let bounds = this.slider_bounds(&key, window); + if let Some(bounds) = bounds { + let value = spec.value_at(event.position.x, bounds); + this.dispatch(spec.handler, HandlerArg::Number(value), cx); + } + }) + }; + let drag_key = key.clone(); + let drag = { + let (spec, key) = (spec.clone(), key.clone()); + cx.listener( + move |this, event: &gpui::DragMoveEvent, _, cx| { + if event.drag(cx).0 != key { + return; + } + let value = spec.value_at(event.event.position.x, event.bounds); + if (value - spec.value).abs() > f64::EPSILON { + this.dispatch(spec.handler, HandlerArg::Number(value), cx); + } + }, + ) + }; + let local = self.local.clone(); + let bounds_key = key.clone(); + div() + .flex() + .flex_col() + .gap(px(4.0)) + .w_full() + .child( + div() + .flex() + .flex_row() + .justify_between() + .gap(px(12.0)) + .text_size(crate::typography::ui_rems(12.0)) + .line_height(crate::typography::ui_rems(18.0)) + .child( + div() + .text_color(theme.text_muted) + .child(spec.label.clone().unwrap_or_default()), + ) + .child(div().text_color(theme.text).child(spec.shown.clone())), + ) + .child( + div() + .id(SharedString::from(format!( + "{}-slider", + self.element_id(&key) + ))) + .h(px(SLIDER_THUMB + 4.0)) + .w_full() + .cursor_pointer() + .role(gpui::Role::Slider) + .on_mouse_down(MouseButton::Left, press) + .on_drag(VisualDrag(drag_key), |_, _, _, cx| cx.new(|_| DragGhost)) + .on_drag_move(drag) + .child( + canvas( + move |bounds, _, _| { + local + .borrow_mut() + .charts + .insert(bounds_key.clone(), ChartGeometry { plot: bounds }); + bounds + }, + move |_, bounds, window, _| { + let mid = bounds.center().y; + let left = bounds.origin.x + px(SLIDER_THUMB / 2.0); + let width = bounds.size.width - px(SLIDER_THUMB); + let at = left + width * fraction; + let bar = + |from: Pixels, to: Pixels, color: Hsla, window: &mut Window| { + window.paint_quad( + gpui::fill( + Bounds::from_corners( + point(from, mid - px(SLIDER_TRACK / 2.0)), + point(to, mid + px(SLIDER_TRACK / 2.0)), + ), + color, + ) + .corner_radii(px(SLIDER_TRACK / 2.0)), + ); + }; + bar(left, left + width, track, window); + bar(left, at, fill, window); + window.paint_quad( + gpui::fill( + Bounds::centered_at( + point(at, mid), + gpui::size(px(SLIDER_THUMB), px(SLIDER_THUMB)), + ), + thumb, + ) + .corner_radii(px(SLIDER_THUMB / 2.0)), + ); + }, + ) + .size_full(), + ), + ) + .into_any_element() + } + + fn slider_bounds(&self, key: &SharedString, _: &Window) -> Option> { + self.local.borrow().charts.get(key).map(|g| g.plot) + } + + /// Track the pointer over a drawn node (canvas, chart, scene). + fn pointer_tracking( + &self, + el: gpui::Stateful, + key: &SharedString, + cx: &mut Context, + ) -> gpui::Stateful { + let (move_key, hover_key) = (key.clone(), key.clone()); + el.on_mouse_move(cx.listener(move |this, event: &MouseMoveEvent, _, cx| { + this.local + .borrow_mut() + .pointers + .insert(move_key.clone(), event.position); + cx.notify(); + })) + .on_hover(cx.listener(move |this, hovered: &bool, _, cx| { + if !*hovered { + this.local.borrow_mut().pointers.remove(&hover_key); + cx.notify(); + } + })) + } + + fn canvas( + &mut self, + node: &zeron_visual::Canvas, + key: SharedString, + cx: &mut Context, + ) -> AnyElement { + let Some(runtime) = self.runtime.clone() else { + return gpui::Empty.into_any_element(); + }; + let (local, generation, time) = (self.local.clone(), self.generation, self.time()); + let (draw, animate) = (node.draw, node.animate); + let paint_palette = Rc::new(Palette::new(Theme::of(cx))); + let cache_key = key.clone(); + let mut el = div() + .id(self.element_id(&format!("{key}-canvas"))) + .w_full() + .h(px(node.height)); + el = self.pointer_tracking(el, &key, cx); + let (down_key, up_key) = (key.clone(), key.clone()); + el = el + .on_mouse_down( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseDownEvent, _, cx| { + this.local.borrow_mut().pressed = Some(down_key.clone()); + cx.notify(); + }), + ) + .on_mouse_up( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + let mut local = this.local.borrow_mut(); + if local.pressed.as_ref() == Some(&up_key) { + local.pressed = None; + } + drop(local); + cx.notify(); + }), + ); + if let Some(handler) = node.on_click { + el = el.cursor_pointer().on_click(cx.listener( + move |this, event: &gpui::ClickEvent, _, cx| { + let origin = this + .local + .borrow() + .charts + .get(&cache_key) + .map(|g| g.plot.origin) + .unwrap_or_default(); + let at = event.position() - origin; + this.dispatch(handler, HandlerArg::Point(at.x.into(), at.y.into()), cx); + }, + )); + } + let geometry_key = key.clone(); + el.child( + canvas( + move |bounds, window, _| { + let mut state = local.borrow_mut(); + state + .charts + .insert(geometry_key.clone(), ChartGeometry { plot: bounds }); + let size = (f32::from(bounds.size.width), f32::from(bounds.size.height)); + let pointer = state.pointers.get(&key).map(|p| { + let at = *p - bounds.origin; + ( + f32::from(at.x), + f32::from(at.y), + state.pressed.as_ref() == Some(&key), + ) + }); + let fresh = state.canvases.get(&key).filter(|cached| { + !animate + && cached.generation == generation + && cached.size == size + && cached.pointer == pointer + }); + if let Some(cached) = fresh { + return Some(cached.commands.clone()); + } + drop(state); + if animate { + window.request_animation_frame(); + } + let drawn = runtime.borrow_mut().draw_canvas( + draw, + size.0, + size.1, + time, + pointer.map(|(x, y, down)| zeron_visual::Pointer { x, y, down }), + ); + let mut state = local.borrow_mut(); + match drawn { + Ok(commands) => { + let commands = Rc::new(commands); + state.canvases.insert( + key.clone(), + CanvasCache { + generation, + size, + pointer, + commands: commands.clone(), + }, + ); + state.draw_error = None; + Some(commands) + } + Err(error) => { + state.draw_error = Some(error.message); + None + } + } + }, + move |bounds, commands, window, cx| { + if let Some(commands) = commands { + paint_commands(&commands, bounds.origin, &paint_palette, window, cx); + } + }, + ) + .size_full(), + ) + .into_any_element() + } +} + +// ---- painting ------------------------------------------------------------- + +fn at(origin: Point, [x, y]: [f32; 2]) -> Point { + point(origin.x + px(x), origin.y + px(y)) +} + +fn stroke_path( + points: impl IntoIterator>, + closed: bool, + width: f32, + dash: Option, + color: Hsla, + window: &mut Window, +) { + let mut builder = PathBuilder::stroke(px(width)); + if let Some(dash) = dash { + builder = builder.dash_array(&[px(dash), px(dash)]); + } + let mut points = points.into_iter(); + let Some(first) = points.next() else { + return; + }; + builder.move_to(first); + for p in points { + builder.line_to(p); + } + if closed { + builder.close(); + } + if let Ok(path) = builder.build() { + window.paint_path(path, color); + } +} + +fn fill_path(points: impl IntoIterator>, color: Hsla, window: &mut Window) { + let mut builder = PathBuilder::fill(); + let mut points = points.into_iter(); + let Some(first) = points.next() else { + return; + }; + builder.move_to(first); + for p in points { + builder.line_to(p); + } + builder.close(); + if let Ok(path) = builder.build() { + window.paint_path(path, color); + } +} + +/// A convex polygon as a triangle fan, skipping tessellation (3D faces). +fn fill_convex(points: &[Point], color: Hsla, window: &mut Window) { + if points.len() < 3 { + return; + } + let mut path = gpui::Path::new(points[0]); + for p in &points[1..] { + path.line_to(*p); + } + window.paint_path(path, color); +} + +#[allow(clippy::too_many_arguments)] +fn paint_text( + text: &str, + at: Point, + size: zeron_visual::TextSize, + w: FontWeight, + mono: bool, + color: Hsla, + align: Align, + baseline: Align, + window: &mut Window, + cx: &mut App, +) -> Pixels { + let theme = Theme::of(cx); + let (font_size, line_height) = text_px(size, window); + let run = TextRun { + len: text.len(), + font: ui_font(theme, mono, w, false), + color, + background_color: None, + underline: None, + strikethrough: None, + }; + let line = window.text_system().shape_line( + SharedString::from(text.to_owned()), + font_size, + &[run], + None, + ); + let width = line.width(); + let x = match align { + Align::Start => at.x, + Align::Center => at.x - width / 2.0, + Align::End => at.x - width, + }; + let y = match baseline { + Align::Start => at.y, + Align::Center => at.y - line_height / 2.0, + Align::End => at.y - line_height, + }; + let _ = line.paint(point(x, y), line_height, TextAlign::Left, None, window, cx); + width +} + +fn paint_commands( + commands: &[DrawCmd], + origin: Point, + palette: &Palette, + window: &mut Window, + cx: &mut App, +) { + for command in commands { + match command { + DrawCmd::Stroke { + points, + closed, + color, + width, + dash, + } => stroke_path( + points.iter().map(|p| at(origin, *p)), + *closed, + *width, + *dash, + palette.color(*color), + window, + ), + DrawCmd::Fill { points, color } => fill_path( + points.iter().map(|p| at(origin, *p)), + palette.color(*color), + window, + ), + DrawCmd::Text(text) => { + paint_text( + &text.text, + at(origin, [text.x, text.y]), + text.size, + weight(text.weight), + text.mono, + palette.color(text.color), + text.align, + text.baseline, + window, + cx, + ); + } + } + } +} + +// ---- 3D ------------------------------------------------------------------- + +impl VisualFrame { + fn scene( + &mut self, + node: &zeron_visual::SceneNode, + key: SharedString, + cx: &mut Context, + ) -> AnyElement { + let Some(runtime) = self.runtime.clone() else { + return gpui::Empty.into_any_element(); + }; + let spin = if cx.reduce_motion() { 0.0 } else { node.spin }; + let orbit = *self + .local + .borrow_mut() + .cameras + .entry(key.clone()) + .or_insert(Orbit { + yaw: node.yaw, + pitch: node.pitch, + spin_epoch: Instant::now(), + drag: None, + }); + let yaw = match orbit.drag { + Some(_) => orbit.yaw, + None => orbit.yaw + spin * orbit.spin_epoch.elapsed().as_secs_f32(), + }; + let camera = zeron_visual::Camera { + yaw, + pitch: orbit.pitch, + zoom: node.zoom, + }; + let moving = (spin != 0.0 && orbit.drag.is_none()) || node.animate; + let (local, generation, time) = (self.local.clone(), self.generation, self.time()); + let (draw, animate) = (node.draw, node.animate); + let paint_palette = Rc::new(Palette::new(Theme::of(cx))); + let mut el = div() + .id(self.element_id(&format!("{key}-scene"))) + .w_full() + .h(px(node.height)); + el = self.pointer_tracking(el, &key, cx); + if node.orbit { + let (down_key, drag_key, move_key, up_key, out_key) = ( + key.clone(), + key.clone(), + key.clone(), + key.clone(), + key.clone(), + ); + let release = move |this: &mut VisualFrame, key: &SharedString| { + if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(key) + && orbit.drag.take().is_some() + { + orbit.spin_epoch = Instant::now(); + } + }; + el = el + .cursor_grab() + .on_mouse_down( + MouseButton::Left, + cx.listener(move |this, event: &gpui::MouseDownEvent, _, cx| { + cx.stop_propagation(); + if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(&down_key) { + orbit.yaw += spin * orbit.spin_epoch.elapsed().as_secs_f32(); + orbit.drag = Some((event.position, orbit.yaw, orbit.pitch)); + } + cx.notify(); + }), + ) + .on_drag(VisualDrag(drag_key), |_, _, _, cx| cx.new(|_| DragGhost)) + .on_drag_move(cx.listener( + move |this, event: &gpui::DragMoveEvent, _, cx| { + if event.drag(cx).0 != move_key { + return; + } + if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(&move_key) + && let Some((start, yaw, pitch)) = orbit.drag + { + let delta = event.event.position - start; + orbit.yaw = yaw - f32::from(delta.x) * 0.01; + orbit.pitch = (pitch + f32::from(delta.y) * 0.01).clamp(-1.5, 1.5); + } + cx.notify(); + }, + )) + .on_mouse_up( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + release(this, &up_key); + cx.notify(); + }), + ) + .on_mouse_up_out( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + release(this, &out_key); + cx.notify(); + }), + ); + } + let pointer_key = key.clone(); + el.child( + canvas( + move |bounds, window, _| { + if moving { + window.request_animation_frame(); + } + let mut state = local.borrow_mut(); + let cached = state + .scenes + .get(&key) + .filter(|c| !animate && c.generation == generation) + .map(|c| c.scene.clone()); + let scene = match cached { + Some(scene) => scene, + None => { + drop(state); + let drawn = runtime.borrow_mut().draw_scene(draw, time); + state = local.borrow_mut(); + match drawn { + Ok(scene) => { + let scene = Rc::new(scene); + state.scenes.insert( + key.clone(), + SceneCache { + generation, + scene: scene.clone(), + }, + ); + state.draw_error = None; + scene + } + Err(error) => { + state.draw_error = Some(error.message); + return None; + } + } + } + }; + let projection = scene.project( + camera, + f32::from(bounds.size.width), + f32::from(bounds.size.height), + ); + let hovered = state.pointers.get(&pointer_key).and_then(|p| { + let at = *p - bounds.origin; + let (x, y) = (f32::from(at.x), f32::from(at.y)); + projection + .hints + .iter() + .map(|([hx, hy], r, text)| ((hx - x).hypot(hy - y), *r, text)) + .filter(|(d, r, _)| *d <= r + 6.0) + .min_by(|a, b| a.0.total_cmp(&b.0)) + .map(|(_, _, text)| text.clone()) + }); + Some((projection, hovered)) + }, + move |bounds, projected, window, cx| { + let Some((projection, hovered)) = projected else { + return; + }; + paint_scene(&projection, bounds.origin, &paint_palette, window, cx); + if let Some(hint) = hovered { + paint_text( + &hint, + bounds.origin, + zeron_visual::TextSize::Sm, + FontWeight::MEDIUM, + false, + paint_palette.text, + Align::Start, + Align::Start, + window, + cx, + ); + } + }, + ) + .size_full(), + ) + .into_any_element() + } +} + +fn paint_scene( + projection: &zeron_visual::Projection, + origin: Point, + palette: &Palette, + window: &mut Window, + cx: &mut App, +) { + for prim in &projection.prims { + match prim { + Prim::Face { + points, + color, + shade, + } => { + // Grow each face half a pixel from its center so neighbours + // overlap and anti-aliasing leaves no seams between them. + let n = points.len() as f32; + let (cx_, cy_) = points + .iter() + .fold((0.0, 0.0), |(x, y), p| (x + p[0] / n, y + p[1] / n)); + let grown: Vec> = points + .iter() + .map(|[x, y]| { + let (dx, dy) = (x - cx_, y - cy_); + let len = dx.hypot(dy).max(1e-3); + at(origin, [x + dx / len * 0.5, y + dy / len * 0.5]) + }) + .collect(); + fill_convex(&grown, shaded(palette.color(*color), *shade), window); + } + Prim::Line { a, b, color, width } => stroke_path( + [at(origin, *a), at(origin, *b)], + false, + *width, + None, + palette.color(*color), + window, + ), + Prim::Dot { + at: center, + radius, + color, + } => { + let points: Vec> = (0..16) + .map(|i| { + let a = std::f32::consts::TAU * i as f32 / 16.0; + at( + origin, + [center[0] + radius * a.cos(), center[1] + radius * a.sin()], + ) + }) + .collect(); + fill_convex(&points, palette.color(*color), window); + } + Prim::Label { + at: anchor, + text, + color, + size, + } => { + paint_text( + text, + at(origin, *anchor), + *size, + FontWeight::NORMAL, + false, + palette.color(*color), + Align::Center, + Align::End, + window, + cx, + ); + } + } + } +} + +// ---- charts --------------------------------------------------------------- + +/// Gaps between the plot and its axis labels, in pixels. +const AXIS_GAP: f32 = 8.0; + +struct ChartPlan { + plot: Bounds, + /// Value-axis domain and ticks. + lo: f64, + hi: f64, + ticks: Vec, + /// Scatter's numeric x domain and ticks. + x_lo: f64, + x_hi: f64, + x_ticks: Vec, + label_stride: usize, + hover: Option, + hover_point: Option<(usize, usize)>, +} + +fn domain(lo: f64, hi: f64, include_zero: bool) -> (f64, f64, Vec) { + let (mut lo, mut hi) = (lo, hi); + if include_zero { + lo = lo.min(0.0); + hi = hi.max(0.0); + } + if !lo.is_finite() || !hi.is_finite() { + return (0.0, 1.0, vec![0.0, 1.0]); + } + if (hi - lo).abs() < f64::EPSILON { + hi = lo + lo.abs().max(1.0); + } + let ticks = zeron_visual::nice_ticks(lo, hi, 4); + let step = if ticks.len() > 1 { + ticks[1] - ticks[0] + } else { + (hi - lo).max(1.0) + }; + let lo = (lo / step).floor() * step; + let hi = (hi / step).ceil() * step; + (lo, hi, zeron_visual::nice_ticks(lo, hi, 4)) +} + +fn value_range(chart: &zeron_visual::Chart) -> (f64, f64) { + let (mut lo, mut hi) = (f64::INFINITY, f64::NEG_INFINITY); + if chart.kind == ChartKind::Scatter { + for (_, y) in chart.series.iter().flat_map(|s| &s.points) { + lo = lo.min(*y); + hi = hi.max(*y); + } + } else if chart.stacked { + let n = chart + .series + .iter() + .map(|s| s.values.len()) + .max() + .unwrap_or(0); + for i in 0..n { + let (mut pos, mut neg) = (0.0, 0.0); + for s in &chart.series { + match s.values.get(i).copied().flatten() { + Some(v) if v >= 0.0 => pos += v, + Some(v) => neg += v, + None => {} + } + } + lo = lo.min(neg); + hi = hi.max(pos); + } + } else { + for v in chart.series.iter().flat_map(|s| s.values.iter().flatten()) { + lo = lo.min(*v); + hi = hi.max(*v); + } + } + (chart.y_min.unwrap_or(lo), chart.y_max.unwrap_or(hi)) +} + +fn categories(chart: &zeron_visual::Chart) -> usize { + chart + .series + .iter() + .map(|s| s.values.len()) + .max() + .unwrap_or(0) + .max(chart.x.len()) +} + +fn measure(text: &str, size: zeron_visual::TextSize, window: &Window, cx: &App) -> f32 { + let theme = Theme::of(cx); + let (font_size, _) = text_px(size, window); + let run = TextRun { + len: text.len(), + font: ui_font(theme, false, FontWeight::NORMAL, false), + color: theme.text, + background_color: None, + underline: None, + strikethrough: None, + }; + f32::from( + window + .text_system() + .shape_line(SharedString::from(text.to_owned()), font_size, &[run], None) + .width(), + ) +} + +impl VisualFrame { + fn chart( + &mut self, + chart: &zeron_visual::Chart, + key: SharedString, + theme: &Theme, + palette: &Palette, + cx: &mut Context, + ) -> AnyElement { + let chart = Rc::new(chart.clone()); + let n = categories(&chart); + // The hovered category, from the plot the last paint laid out. + let (hover, hover_point) = { + let local = self.local.borrow(); + let geometry = local.charts.get(&key).copied(); + let pointer = local.pointers.get(&key).copied(); + match (geometry, pointer) { + (Some(g), Some(p)) if g.plot.size.width > px(0.0) => { + let along = if chart.horizontal && chart.kind == ChartKind::Bar { + (p.y - g.plot.origin.y) / g.plot.size.height + } else { + (p.x - g.plot.origin.x) / g.plot.size.width + }; + let ix = if chart.kind == ChartKind::Bar || n <= 1 { + (along * n as f32).floor() + } else { + (along * (n - 1) as f32).round() + }; + let hover = (0.0..n as f32).contains(&ix).then_some(ix as usize); + ( + hover.filter(|_| chart.kind != ChartKind::Scatter), + (chart.kind == ChartKind::Scatter) + .then(|| nearest_point(&chart, g.plot, p)) + .flatten(), + ) + } + _ => (None, None), + } + }; + + // Legend and readout: plain type above the plot, the hovered values + // inline with their series swatches. + let readout_label = hover.and_then(|i| chart.x.get(i).cloned()); + let mut legend = div() + .flex() + .flex_row() + .flex_wrap() + .items_center() + .gap_x(px(14.0)) + .gap_y(px(2.0)) + .text_size(crate::typography::ui_rems(12.0)) + .line_height(crate::typography::ui_rems(18.0)) + .text_color(theme.text_muted) + .min_h(crate::typography::ui_rems(18.0)); + if let Some(label) = &readout_label { + legend = legend.child( + div() + .text_color(theme.text) + .font_weight(FontWeight::MEDIUM) + .child(label.clone()), + ); + } + if let Some((s, p)) = hover_point { + let (x, y) = chart.series[s].points[p]; + legend = legend.child(div().text_color(theme.text).child(format!( + "{} · {}", + chart.x_format.format(x), + chart.format.format(y) + ))); + } + let show_legend = chart.legend + && (chart.series.len() > 1 || hover.is_some()) + && chart.series.iter().any(|s| !s.name.is_empty()); + if show_legend || hover.is_some() { + for series in chart.series.iter() { + let value = hover.and_then(|i| series.values.get(i).copied().flatten()); + let mut entry = div() + .flex() + .flex_row() + .items_center() + .gap(px(6.0)) + .child( + div() + .flex_none() + .size(px(8.0)) + .rounded(px(2.0)) + .bg(palette.color(series.color)), + ) + .child(series.name.clone()); + if let Some(value) = value { + entry = entry.child( + div() + .text_color(theme.text) + .child(chart.format.format(value)), + ); + } + legend = legend.child(entry); + } + } + + let local = self.local.clone(); + let paint_palette = Rc::new(Palette::new(theme)); + let (plan_key, paint_chart) = (key.clone(), chart.clone()); + let mut plot = div() + .id(self.element_id(&format!("{key}-chart"))) + .w_full() + .h(px(chart.height)); + plot = self.pointer_tracking(plot, &key, cx); + div() + .flex() + .flex_col() + .gap(px(8.0)) + .w_full() + .child(legend) + .child( + plot.child( + canvas( + move |bounds, window, cx| { + let plan = plan_chart(&chart, bounds, hover, hover_point, window, cx); + local + .borrow_mut() + .charts + .insert(plan_key.clone(), ChartGeometry { plot: plan.plot }); + plan + }, + move |_, plan, window, cx| { + paint_chart_plan(&paint_chart, &plan, &paint_palette, window, cx) + }, + ) + .size_full(), + ), + ) + .into_any_element() + } +} + +fn nearest_point( + chart: &zeron_visual::Chart, + plot: Bounds, + pointer: Point, +) -> Option<(usize, usize)> { + let (x_lo, x_hi) = x_extent(chart); + let (lo, hi) = value_range(chart); + let (x_lo, x_hi, _) = domain(x_lo, x_hi, false); + let (lo, hi, _) = domain(lo, hi, false); + let mut best = None; + for (s, series) in chart.series.iter().enumerate() { + for (p, (x, y)) in series.points.iter().enumerate() { + let sx = plot.origin.x + plot.size.width * ((x - x_lo) / (x_hi - x_lo)) as f32; + let sy = plot.origin.y + plot.size.height * (1.0 - ((y - lo) / (hi - lo)) as f32); + let d = f32::from((sx - pointer.x).abs()).hypot(f32::from((sy - pointer.y).abs())); + if d < 12.0 && best.is_none_or(|(bd, _, _)| d < bd) { + best = Some((d, s, p)); + } + } + } + best.map(|(_, s, p)| (s, p)) +} + +fn x_extent(chart: &zeron_visual::Chart) -> (f64, f64) { + chart + .series + .iter() + .flat_map(|s| &s.points) + .fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), (x, _)| { + (lo.min(*x), hi.max(*x)) + }) +} + +fn plan_chart( + chart: &zeron_visual::Chart, + bounds: Bounds, + hover: Option, + hover_point: Option<(usize, usize)>, + window: &mut Window, + cx: &mut App, +) -> ChartPlan { + use zeron_visual::TextSize::Xs; + let (_, tick_line) = text_px(Xs, window); + let tick_line = f32::from(tick_line); + let scatter = chart.kind == ChartKind::Scatter; + let bars = chart.kind == ChartKind::Bar; + let horizontal = bars && chart.horizontal; + let (lo, hi) = value_range(chart); + let (lo, hi, ticks) = domain(lo, hi, !scatter); + let (x_lo, x_hi, x_ticks) = if scatter { + let (a, b) = x_extent(chart); + domain(a, b, false) + } else { + (0.0, 1.0, Vec::new()) + }; + let n = categories(chart).max(1); + // The left gutter fits the widest value tick, or the category labels on + // a horizontal bar chart. + let left = if horizontal { + chart + .x + .iter() + .map(|l| measure(l, Xs, window, cx)) + .fold(0.0, f32::max) + .min(f32::from(bounds.size.width) * 0.35) + } else { + ticks + .iter() + .map(|t| measure(&chart.format.format(*t), Xs, window, cx)) + .fold(0.0, f32::max) + } + AXIS_GAP; + let top = tick_line / 2.0; + let bottom = tick_line + AXIS_GAP / 2.0; + let right = if horizontal || scatter { 12.0 } else { 4.0 }; + let plot = Bounds::from_corners( + bounds.origin + point(px(left), px(top)), + point( + bounds.origin.x + bounds.size.width - px(right), + bounds.origin.y + bounds.size.height - px(bottom), + ), + ); + // Thin the category labels until they don't collide. + let label_stride = if horizontal || scatter { + 1 + } else { + let widest = chart + .x + .iter() + .map(|l| measure(l, Xs, window, cx)) + .fold(0.0, f32::max) + + 12.0; + let band = f32::from(plot.size.width) / n as f32; + ((widest / band.max(1.0)).ceil() as usize).max(1) + }; + ChartPlan { + plot, + lo, + hi, + ticks, + x_lo, + x_hi, + x_ticks, + label_stride, + hover, + hover_point, + } +} + +fn paint_chart_plan( + chart: &zeron_visual::Chart, + plan: &ChartPlan, + palette: &Palette, + window: &mut Window, + cx: &mut App, +) { + use zeron_visual::TextSize::Xs; + let plot = plan.plot; + if plot.size.width <= px(4.0) || plot.size.height <= px(4.0) { + return; + } + let horizontal = chart.kind == ChartKind::Bar && chart.horizontal; + let n = categories(chart).max(1); + let (pw, ph) = (f32::from(plot.size.width), f32::from(plot.size.height)); + let (ox, oy) = (f32::from(plot.origin.x), f32::from(plot.origin.y)); + let value_t = |v: f64| ((v - plan.lo) / (plan.hi - plan.lo)) as f32; + // Value -> screen along the value axis. + let vy = |v: f64| oy + ph * (1.0 - value_t(v)); + let vx = |v: f64| ox + pw * value_t(v); + let grid = palette.border; + let muted = palette.muted; + let hover_wash = Theme::of(cx).glass_hover(); + let p = |x: f32, y: f32| point(px(x), px(y)); + + // Gridlines and value ticks. + for tick in &plan.ticks { + let label = chart.format.format(*tick); + if horizontal { + let x = vx(*tick); + stroke_path([p(x, oy), p(x, oy + ph)], false, 1.0, None, grid, window); + paint_text( + &label, + p(x, oy + ph + AXIS_GAP / 2.0), + Xs, + FontWeight::NORMAL, + false, + muted, + Align::Center, + Align::Start, + window, + cx, + ); + } else { + let y = vy(*tick); + stroke_path([p(ox, y), p(ox + pw, y)], false, 1.0, None, grid, window); + paint_text( + &label, + p(ox - AXIS_GAP, y), + Xs, + FontWeight::NORMAL, + false, + muted, + Align::End, + Align::Center, + window, + cx, + ); + } + } + + match chart.kind { + ChartKind::Bar => { + let band = if horizontal { ph } else { pw } / n as f32; + if let Some(i) = plan.hover { + let start = if horizontal { oy } else { ox } + band * i as f32; + let rect = if horizontal { + Bounds::from_corners(p(ox, start), p(ox + pw, start + band)) + } else { + Bounds::from_corners(p(start, oy), p(start + band, oy + ph)) + }; + window.paint_quad(gpui::fill(rect, hover_wash).corner_radii(px(4.0))); + } + let groups = if chart.stacked { + 1 + } else { + chart.series.len().max(1) + }; + let inner = band * 0.72; + let bar = (inner / groups as f32).max(1.0); + for i in 0..n { + let start = band * i as f32 + (band - inner) / 2.0; + let (mut pos, mut neg) = (0.0f64, 0.0f64); + for (s, series) in chart.series.iter().enumerate() { + let Some(v) = series.values.get(i).copied().flatten() else { + continue; + }; + let (from, to) = if chart.stacked { + if v >= 0.0 { + pos += v; + (pos - v, pos) + } else { + neg += v; + (neg - v, neg) + } + } else { + (0.0f64.clamp(plan.lo, plan.hi), v) + }; + let offset = if chart.stacked { 0.0 } else { bar * s as f32 }; + let gap = if groups > 1 && !chart.stacked { + 1.0 + } else { + 0.0 + }; + let rect = if horizontal { + let y = oy + start + offset; + let (a, b) = (vx(from), vx(to)); + Bounds::from_corners(p(a.min(b), y), p(a.max(b), y + bar - gap)) + } else { + let x = ox + start + offset; + let (a, b) = (vy(from), vy(to)); + Bounds::from_corners(p(x, a.min(b)), p(x + bar - gap, a.max(b))) + }; + let radius = if chart.stacked { 0.0 } else { 2.0 }; + window.paint_quad( + gpui::fill(rect, palette.color(series.color)).corner_radii(px(radius)), + ); + } + } + // Category labels. + for (i, label) in chart.x.iter().enumerate().take(n) { + if i % plan.label_stride != 0 { + continue; + } + let center = band * (i as f32 + 0.5); + if horizontal { + paint_text( + label, + p(ox - AXIS_GAP, oy + center), + Xs, + FontWeight::NORMAL, + false, + muted, + Align::End, + Align::Center, + window, + cx, + ); + } else { + paint_text( + label, + p(ox + center, oy + ph + AXIS_GAP / 2.0), + Xs, + FontWeight::NORMAL, + false, + muted, + Align::Center, + Align::Start, + window, + cx, + ); + } + } + } + ChartKind::Line | ChartKind::Area => { + let x_at = |i: usize| { + if n <= 1 { + ox + pw / 2.0 + } else { + ox + pw * i as f32 / (n - 1) as f32 + } + }; + if let Some(i) = plan.hover { + let x = x_at(i); + stroke_path( + [p(x, oy), p(x, oy + ph)], + false, + 1.0, + None, + Theme::of(cx).border_strong, + window, + ); + } + let base = vy(0.0f64.clamp(plan.lo, plan.hi)); + for series in &chart.series { + let color = palette.color(series.color); + // Gaps split a series into runs. + let mut runs: Vec>> = vec![Vec::new()]; + for (i, v) in series.values.iter().enumerate() { + match v { + Some(v) => runs.last_mut().unwrap().push(p(x_at(i), vy(*v))), + None => runs.push(Vec::new()), + } + } + for run in runs.iter().filter(|r| !r.is_empty()) { + if chart.kind == ChartKind::Area && run.len() > 1 { + let mut area = run.clone(); + area.push(point(run.last().unwrap().x, px(base))); + area.push(point(run[0].x, px(base))); + fill_path( + area, + Hsla { + a: color.a * 0.22, + ..color + }, + window, + ); + } + if run.len() == 1 { + fill_convex(&dot(run[0], 2.5), color, window); + } else { + stroke_path(run.iter().copied(), false, 1.75, None, color, window); + } + } + if let Some(i) = plan.hover + && let Some(Some(v)) = series.values.get(i) + { + fill_convex(&dot(p(x_at(i), vy(*v)), 3.5), color, window); + } + } + for (i, label) in chart.x.iter().enumerate().take(n) { + if i % plan.label_stride != 0 { + continue; + } + let align = if n > 1 && i == 0 { + Align::Start + } else if n > 1 && i == n - 1 { + Align::End + } else { + Align::Center + }; + paint_text( + label, + p(x_at(i), oy + ph + AXIS_GAP / 2.0), + Xs, + FontWeight::NORMAL, + false, + muted, + align, + Align::Start, + window, + cx, + ); + } + } + ChartKind::Scatter => { + let sx = |x: f64| ox + pw * ((x - plan.x_lo) / (plan.x_hi - plan.x_lo)) as f32; + for tick in &plan.x_ticks { + let x = sx(*tick); + stroke_path([p(x, oy), p(x, oy + ph)], false, 1.0, None, grid, window); + paint_text( + &chart.x_format.format(*tick), + p(x, oy + ph + AXIS_GAP / 2.0), + Xs, + FontWeight::NORMAL, + false, + muted, + Align::Center, + Align::Start, + window, + cx, + ); + } + for (s, series) in chart.series.iter().enumerate() { + let color = palette.color(series.color); + for (ix, (x, y)) in series.points.iter().enumerate() { + let r = if plan.hover_point == Some((s, ix)) { + 5.0 + } else { + 3.0 + }; + fill_convex(&dot(p(sx(*x), vy(*y)), r), color, window); + } + } + } + } +} + +fn dot(center: Point, radius: f32) -> Vec> { + (0..14) + .map(|i| { + let a = std::f32::consts::TAU * i as f32 / 14.0; + center + point(px(radius * a.cos()), px(radius * a.sin())) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chart_series_drops_the_fixed_hue_nearest_the_accent() { + let theme = Theme::dark(); + let series = chart_series(&theme); + assert_eq!(series[0], theme.accent_strong); + // Six distinct colors. + for (i, a) in series.iter().enumerate() { + for b in &series[i + 1..] { + assert_ne!(a, b); + } + } + } + + #[test] + fn every_token_resolves_against_the_theme() { + for theme in [Theme::dark(), Theme::light()] { + let palette = Palette::new(&theme); + for name in zeron_visual::TOKEN_NAMES { + let token = Token::parse(name).unwrap(); + let color = palette.token(token); + assert!(color.a > 0.0, "{name}"); + } + assert_eq!(palette.token(Token::Text), theme.text); + let half = palette.color(zeron_visual::Color::token(Token::Accent).with_alpha(0.5)); + assert!((half.a - theme.accent.a * 0.5).abs() < 1e-6); + } + } + + #[test] + fn domains_include_zero_for_bars_and_land_on_ticks() { + let (lo, hi, ticks) = domain(12.0, 87.0, true); + assert_eq!(lo, 0.0); + assert_eq!(hi, 100.0); + assert_eq!(ticks.first(), Some(&0.0)); + assert_eq!(ticks.last(), Some(&100.0)); + let (lo, hi, _) = domain(-3.0, 7.0, false); + assert!(lo <= -3.0 && hi >= 7.0); + } +} diff --git a/crates/visual/Cargo.toml b/crates/visual/Cargo.toml new file mode 100644 index 0000000000..5cc323039a --- /dev/null +++ b/crates/visual/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "zeron-visual" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "Sandboxed Luau runtime for agent visuals: scripts build a native element tree, canvas drawings, and 3D scenes" + +[dependencies] +mlua = { version = "0.12", features = ["luau"] } +thiserror.workspace = true diff --git a/crates/visual/src/draw.rs b/crates/visual/src/draw.rs new file mode 100644 index 0000000000..f08402b37f --- /dev/null +++ b/crates/visual/src/draw.rs @@ -0,0 +1,421 @@ +//! `ui.canvas` drawing: the draw function gets a `g` object whose methods +//! record vector commands in design pixels, origin top-left. The desktop +//! paints them with GPUI paths and shaped text, so a canvas is as crisp and +//! as theme-bound as the rest of the reply. + +use mlua::{Table, UserData, UserDataFields, UserDataMethods, Value}; + +use crate::VisualError; +use crate::node::{parse_color, value_text}; +use crate::style::{Align, Color, TextSize, Token, Weight}; + +pub const MAX_COMMANDS: usize = 40_000; +const MAX_POINTS: usize = 200_000; + +#[derive(Debug, Clone, PartialEq)] +pub enum DrawCmd { + /// An open or closed stroked path. + Stroke { + points: Vec<[f32; 2]>, + closed: bool, + color: Color, + width: f32, + dash: Option, + }, + /// A filled polygon (any simple polygon, convex or not). + Fill { + points: Vec<[f32; 2]>, + color: Color, + }, + Text(DrawText), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct DrawText { + pub x: f32, + pub y: f32, + pub text: String, + pub size: TextSize, + pub weight: Weight, + pub color: Color, + pub mono: bool, + pub align: Align, + pub baseline: Align, +} + +/// The pointer, in canvas pixels, while it is over the canvas. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct Pointer { + pub x: f32, + pub y: f32, + pub down: bool, +} + +pub struct Graphics { + pub width: f32, + pub height: f32, + pub time: f64, + pub pointer: Option, + pub commands: Vec, + points: usize, +} + +impl Graphics { + pub fn new(width: f32, height: f32, time: f64, pointer: Option) -> Self { + Self { + width, + height, + time, + pointer, + commands: Vec::new(), + points: 0, + } + } + + fn push(&mut self, command: DrawCmd) -> mlua::Result<()> { + if self.commands.len() >= MAX_COMMANDS { + return Err(lua_error(format!( + "a canvas draws at most {MAX_COMMANDS} shapes per frame" + ))); + } + self.points += match &command { + DrawCmd::Stroke { points, .. } | DrawCmd::Fill { points, .. } => points.len(), + DrawCmd::Text(_) => 1, + }; + if self.points > MAX_POINTS { + return Err(lua_error(format!( + "a canvas draws at most {MAX_POINTS} points per frame" + ))); + } + self.commands.push(command); + Ok(()) + } +} + +pub(crate) fn lua_error(message: impl Into) -> mlua::Error { + mlua::Error::RuntimeError(message.into()) +} + +fn to_lua(error: VisualError) -> mlua::Error { + lua_error(error.to_string()) +} + +/// Shape options: `{ color = ..., fill = ..., stroke = ..., width = ..., +/// dash = ... }`. +#[derive(Default)] +pub(crate) struct Paint { + pub fill: Option, + pub stroke: Option, + pub width: f32, + pub dash: Option, +} + +impl Paint { + /// `default_fill` decides what a bare `color` means: a fill for shapes, + /// a stroke for lines. + pub fn parse(options: Option, default_fill: bool) -> mlua::Result { + let mut paint = Paint { + width: 1.0, + ..Default::default() + }; + let Some(options) = options else { + let color = Color::token(Token::Text); + if default_fill { + paint.fill = Some(color); + } else { + paint.stroke = Some(color); + } + return Ok(paint); + }; + let color = parse_color(options.raw_get::("color")?).map_err(to_lua)?; + paint.fill = parse_color(options.raw_get::("fill")?).map_err(to_lua)?; + paint.stroke = parse_color(options.raw_get::("stroke")?).map_err(to_lua)?; + if let Some(color) = color { + if default_fill && paint.fill.is_none() { + paint.fill = Some(color); + } else if paint.stroke.is_none() { + paint.stroke = Some(color); + } + } + if paint.fill.is_none() && paint.stroke.is_none() { + let color = Color::token(Token::Text); + if default_fill { + paint.fill = Some(color); + } else { + paint.stroke = Some(color); + } + } + if let Some(alpha) = options.raw_get::>("alpha")? { + paint.fill = paint.fill.map(|c| c.with_alpha(alpha as f32)); + paint.stroke = paint.stroke.map(|c| c.with_alpha(alpha as f32)); + } + paint.width = options + .raw_get::>("width")? + .unwrap_or(1.0) + .clamp(0.25, 32.0) as f32; + paint.dash = options + .raw_get::>("dash")? + .map(|d| d.clamp(1.0, 64.0) as f32); + Ok(paint) + } +} + +/// Points as a flat `{x1, y1, x2, y2, ...}` list or `{{x, y}, ...}` pairs. +pub(crate) fn points(table: &Table) -> mlua::Result> { + let len = table.raw_len(); + let mut out = Vec::with_capacity(len / 2 + 1); + match table.raw_get::(1)? { + Value::Table(_) => { + for pair in table.sequence_values::
() { + let pair = pair?; + out.push([ + pair.raw_get::(1)? as f32, + pair.raw_get::(2)? as f32, + ]); + } + } + _ => { + for ix in (1..len).step_by(2) { + out.push([ + table.raw_get::(ix)? as f32, + table.raw_get::(ix + 1)? as f32, + ]); + } + } + } + if out.iter().any(|[x, y]| !x.is_finite() || !y.is_finite()) { + return Err(lua_error("a point is NaN or infinite")); + } + Ok(out) +} + +/// `segments` points around an ellipse arc from `a0` to `a1` (radians, +/// clockwise from 3 o'clock, since y points down). +pub(crate) fn arc_points(cx: f32, cy: f32, r: f32, a0: f32, a1: f32) -> Vec<[f32; 2]> { + let sweep = a1 - a0; + let segments = ((sweep.abs() * r.max(1.0)).sqrt() * 2.5).clamp(6.0, 160.0) as usize; + (0..=segments) + .map(|i| { + let a = a0 + sweep * i as f32 / segments as f32; + [cx + r * a.cos(), cy + r * a.sin()] + }) + .collect() +} + +impl Graphics { + fn shape(&mut self, points: Vec<[f32; 2]>, paint: &Paint) -> mlua::Result<()> { + if let Some(fill) = paint.fill { + self.push(DrawCmd::Fill { + points: points.clone(), + color: fill, + })?; + } + if let Some(stroke) = paint.stroke { + self.push(DrawCmd::Stroke { + points, + closed: true, + color: stroke, + width: paint.width, + dash: paint.dash, + })?; + } + Ok(()) + } +} + +fn rect_points(x: f32, y: f32, w: f32, h: f32, radius: f32) -> Vec<[f32; 2]> { + let (x, w) = if w < 0.0 { (x + w, -w) } else { (x, w) }; + let (y, h) = if h < 0.0 { (y + h, -h) } else { (y, h) }; + let r = radius.clamp(0.0, w.min(h) / 2.0); + if r < 0.25 { + return vec![[x, y], [x + w, y], [x + w, y + h], [x, y + h]]; + } + use std::f32::consts::{FRAC_PI_2, PI}; + let mut out = Vec::new(); + out.extend(arc_points(x + w - r, y + r, r, -FRAC_PI_2, 0.0)); + out.extend(arc_points(x + w - r, y + h - r, r, 0.0, FRAC_PI_2)); + out.extend(arc_points(x + r, y + h - r, r, FRAC_PI_2, PI)); + out.extend(arc_points(x + r, y + r, r, PI, PI + FRAC_PI_2)); + out +} + +impl UserData for Graphics { + fn add_fields>(fields: &mut F) { + fields.add_field_method_get("width", |_, g| Ok(g.width)); + fields.add_field_method_get("height", |_, g| Ok(g.height)); + fields.add_field_method_get("time", |_, g| Ok(g.time)); + fields.add_field_method_get("pointer", |lua, g| match g.pointer { + Some(p) => { + let t = lua.create_table()?; + t.set("x", p.x)?; + t.set("y", p.y)?; + t.set("down", p.down)?; + Ok(Value::Table(t)) + } + None => Ok(Value::Nil), + }); + } + + fn add_methods>(methods: &mut M) { + methods.add_method_mut( + "line", + |_, g, (x1, y1, x2, y2, options): (f32, f32, f32, f32, Option
)| { + let paint = Paint::parse(options, false)?; + g.push(DrawCmd::Stroke { + points: vec![[x1, y1], [x2, y2]], + closed: false, + color: paint.stroke.or(paint.fill).unwrap(), + width: paint.width, + dash: paint.dash, + }) + }, + ); + methods.add_method_mut( + "polyline", + |_, g, (pts, options): (Table, Option
)| { + let paint = Paint::parse(options, false)?; + let points = points(&pts)?; + if points.len() < 2 { + return Ok(()); + } + g.push(DrawCmd::Stroke { + points, + closed: false, + color: paint.stroke.or(paint.fill).unwrap(), + width: paint.width, + dash: paint.dash, + }) + }, + ); + methods.add_method_mut("polygon", |_, g, (pts, options): (Table, Option
)| { + let paint = Paint::parse(options, true)?; + let points = points(&pts)?; + if points.len() < 3 { + return Ok(()); + } + g.shape(points, &paint) + }); + methods.add_method_mut( + "rect", + |_, g, (x, y, w, h, options): (f32, f32, f32, f32, Option
)| { + let radius = match &options { + Some(o) => o.raw_get::>("radius")?.unwrap_or(0.0), + None => 0.0, + }; + let paint = Paint::parse(options, true)?; + g.shape(rect_points(x, y, w, h, radius), &paint) + }, + ); + methods.add_method_mut( + "circle", + |_, g, (x, y, r, options): (f32, f32, f32, Option
)| { + let paint = Paint::parse(options, true)?; + let mut pts = arc_points(x, y, r.abs(), 0.0, std::f32::consts::TAU); + pts.pop(); + g.shape(pts, &paint) + }, + ); + // An arc stroke, or with `inner` (a radius) / `fill`, a pie or donut + // sector. + methods.add_method_mut( + "arc", + |_, g, (x, y, r, a0, a1, options): (f32, f32, f32, f32, f32, Option
)| { + let inner = match &options { + Some(o) => o.raw_get::>("inner")?, + None => None, + }; + let sector = inner.is_some() + || options + .as_ref() + .is_some_and(|o| o.raw_get::("fill").is_ok_and(|v| !v.is_nil())); + let paint = Paint::parse(options, sector)?; + let outer = arc_points(x, y, r.abs(), a0, a1); + if !sector { + return g.push(DrawCmd::Stroke { + points: outer, + closed: false, + color: paint.stroke.or(paint.fill).unwrap(), + width: paint.width, + dash: paint.dash, + }); + } + let mut pts = outer; + match inner.filter(|r| *r > 0.0) { + Some(inner) => { + let mut back = arc_points(x, y, inner, a0, a1); + back.reverse(); + pts.extend(back); + } + None => pts.push([x, y]), + } + g.shape(pts, &paint) + }, + ); + methods.add_method_mut( + "text", + |_, g, (x, y, text, options): (f32, f32, Value, Option
)| { + let mut draw = DrawText { + x, + y, + text: value_text(&text), + size: TextSize::Sm, + weight: Weight::Regular, + color: Color::token(Token::Text), + mono: false, + align: Align::Start, + baseline: Align::Start, + }; + if let Some(o) = options { + if let Some(size) = o.raw_get::>("size")? { + draw.size = TextSize::parse(&size).map_err(to_lua)?; + } + match o.raw_get::("weight")? { + Value::Integer(n) => draw.weight = Weight::from_number(n as f64), + Value::Number(n) => draw.weight = Weight::from_number(n), + Value::String(s) => { + draw.weight = Weight::parse(&s.to_string_lossy()).map_err(to_lua)? + } + _ => {} + } + if let Some(color) = + parse_color(o.raw_get::("color")?).map_err(to_lua)? + { + draw.color = color; + } + draw.mono = o.raw_get::>("mono")?.unwrap_or(false); + if let Some(align) = o.raw_get::>("align")? { + draw.align = Align::parse(&align).map_err(to_lua)?; + } + if let Some(baseline) = o.raw_get::>("baseline")? { + draw.baseline = Align::parse(&baseline).map_err(to_lua)?; + } + } + if draw.text.is_empty() { + return Ok(()); + } + draw.text = draw.text.replace('\n', " "); + g.push(DrawCmd::Text(draw)) + }, + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rounded_rects_and_sectors_are_closed_polygons() { + let plain = rect_points(0.0, 0.0, 10.0, 4.0, 0.0); + assert_eq!(plain.len(), 4); + let rounded = rect_points(0.0, 0.0, 10.0, 4.0, 2.0); + assert!(rounded.len() > 8); + assert!( + rounded + .iter() + .all(|[x, y]| *x >= -0.01 && *x <= 10.01 && *y >= -0.01 && *y <= 4.01) + ); + let negative = rect_points(10.0, 4.0, -10.0, -4.0, 0.0); + assert_eq!(negative[0], [0.0, 0.0]); + } +} diff --git a/crates/visual/src/lib.rs b/crates/visual/src/lib.rs new file mode 100644 index 0000000000..4d2e505fda --- /dev/null +++ b/crates/visual/src/lib.rs @@ -0,0 +1,536 @@ +//! Agent visuals as sandboxed Luau programs. +//! +//! An agent's `show_visual` call carries a Luau script. The script builds its +//! data once and returns a view function; the view returns a tree of `ui.*` +//! nodes (layout, text, stats, tables, charts, controls, canvases, 3D scenes). +//! Zeron renders that tree with native elements, so a visual scrolls, clips, +//! takes input, and sits on frosted glass exactly like the transcript around +//! it, on every platform. Interactivity is Lua: controls call handlers that +//! change the script's state, and Zeron renders the view again. +//! +//! The style rules are the API: colors are theme tokens, text sizes are steps +//! on the transcript's type scale, the font is always the interface font, and +//! layout nodes have no fills, borders, or shadows to reach for. +//! +//! Scripts run in a Luau sandbox with no I/O, a memory cap, and a time budget +//! per call, so a runaway script fails with an error instead of freezing the +//! app. `math.random` is seeded from the visual's id before every call, so a +//! script's random data is the same on every render and every device. + +mod draw; +mod node; +mod scene; +mod style; + +use std::cell::{Cell, RefCell}; +use std::rc::Rc; +use std::time::{Duration, Instant}; + +use mlua::{Function, Lua, LuaOptions, StdLib, Table, Value, VmState}; + +pub use draw::{DrawCmd, DrawText, Pointer}; +pub use node::{ + Axis, Canvas, Chart, ChartKind, Column, HandlerId, Node, NodeKind, NumberFormat, SceneNode, + Series, Span, Stack, Table_ as TableNode, Text, +}; +pub use scene::{Camera, Prim, Projection, Scene}; +pub use style::{Align, CHART_SERIES, Color, Justify, TOKEN_NAMES, TextSize, Token, Weight}; + +/// Largest script `show_visual` accepts. +pub const MAX_SCRIPT_BYTES: usize = 256 * 1024; +const MEMORY_LIMIT: usize = 64 * 1024 * 1024; +/// Setup and view calls may take this long before they fail. +const CALL_BUDGET: Duration = Duration::from_millis(250); +/// A canvas or scene draw runs per frame while animating. +const DRAW_BUDGET: Duration = Duration::from_millis(60); + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{message}")] +pub struct VisualError { + pub message: String, +} + +impl VisualError { + pub fn script(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl From for VisualError { + fn from(error: mlua::Error) -> Self { + Self::script(describe(&error)) + } +} + +/// The innermost message of a Lua error, without mlua's wrapping. +fn describe(error: &mlua::Error) -> String { + match error { + // A Rust callback's error carries no position; take the script line + // that called it from the traceback. + mlua::Error::CallbackError { cause, traceback } => { + let message = describe(cause); + match traceback + .lines() + .find_map(|line| line.trim().strip_prefix("visual:")) + .and_then(|rest| rest.split(':').next()) + .filter(|_| !message.starts_with("visual:")) + { + Some(line) => format!("visual:{line}: {message}"), + None => message, + } + } + mlua::Error::RuntimeError(message) => message.clone(), + mlua::Error::SyntaxError { message, .. } => message.clone(), + mlua::Error::MemoryError(_) => { + format!("the script used more than {} MB", MEMORY_LIMIT >> 20) + } + mlua::Error::WithContext { context, cause } => format!("{context}: {}", describe(cause)), + other => node::short_error(other), + } +} + +/// Arguments a node passes to its handler. +#[derive(Debug, Clone, PartialEq)] +pub enum HandlerArg { + None, + Text(String), + Bool(bool), + Number(f64), + Point(f32, f32), +} + +/// One visual's live program. +pub struct Visual { + lua: Lua, + view: Function, + handlers: Vec, + randomseed: Function, + seed: i64, + deadline: Rc>>, + logs: Rc>>, +} + +const MAX_LOG_LINES: usize = 200; + +impl Visual { + /// Compile and run a script's setup. `seed` keeps its random numbers + /// stable (derive it from the visual's id). + pub fn load(source: &str, seed: u64) -> Result { + if source.len() > MAX_SCRIPT_BYTES { + return Err(VisualError::script(format!( + "the script is over {} KB", + MAX_SCRIPT_BYTES / 1024 + ))); + } + let lua = Lua::new_with( + StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8 | StdLib::BIT, + LuaOptions::default(), + )?; + lua.set_memory_limit(MEMORY_LIMIT)?; + + let deadline: Rc>> = Rc::default(); + let ticks = Cell::new(0u32); + let watched = deadline.clone(); + lua.set_interrupt(move |_| { + // Reading the clock on every safepoint is measurable; every 256th + // is plenty to stop a runaway loop within a few milliseconds. + let tick = ticks.get().wrapping_add(1); + ticks.set(tick); + if tick.is_multiple_of(256) + && let Some(deadline) = watched.get() + && Instant::now() > deadline + { + return Err(mlua::Error::RuntimeError( + "the script ran past its time budget (an endless loop, or too much work \ + per render; precompute data at the top level)" + .into(), + )); + } + Ok(VmState::Continue) + }); + + let globals = lua.globals(); + let logs: Rc>> = Rc::default(); + let sink = logs.clone(); + globals.set( + "print", + lua.create_function(move |_, args: mlua::Variadic| { + let mut logs = sink.borrow_mut(); + if logs.len() < MAX_LOG_LINES { + let line: Vec = args.iter().map(node::value_text).collect(); + logs.push(line.join("\t")); + } + Ok(()) + })?, + )?; + let ui: Table = lua + .load(include_str!("prelude.luau")) + .set_name("=ui") + .eval()?; + ui.set( + "fmt", + lua.create_function(|_, (value, options): (f64, Option
)| { + let mut format = NumberFormat::default(); + if let Some(o) = options { + format.prefix = o.get::>("prefix")?.unwrap_or_default(); + format.unit = o.get::>("unit")?.unwrap_or_default(); + format.decimals = o + .get::>("decimals")? + .map(|d| d.clamp(0.0, 6.0) as u8); + } + Ok(format.format(value)) + })?, + )?; + ui.set( + "ticks", + lua.create_function(|_, (lo, hi, count): (f64, f64, Option)| { + Ok(nice_ticks(lo, hi, count.unwrap_or(5))) + })?, + )?; + globals.set("ui", ui)?; + let math: Table = globals.get("math")?; + let randomseed: Function = math.get("randomseed")?; + lua.sandbox(true)?; + + let seed = (seed & 0x7fff_ffff) as i64; + let visual_deadline = deadline.clone(); + let call = |f: &dyn Fn() -> mlua::Result| { + visual_deadline.set(Some(Instant::now() + CALL_BUDGET)); + let result = f(); + visual_deadline.set(None); + result + }; + randomseed.call::<()>(seed)?; + let returned = call(&|| lua.load(source).set_name("=visual").eval::())?; + let view = match returned { + Value::Function(view) => view, + Value::Table(tree) => lua.create_function(move |_, ()| Ok(tree.clone()))?, + Value::Nil => { + return Err(VisualError::script( + "the script must end with `return function() return ui.column { ... } end` \ + (or return a node)", + )); + } + other => { + return Err(VisualError::script(format!( + "the script returned a {}; return a view function or a ui node", + other.type_name() + ))); + } + }; + Ok(Self { + lua, + view, + handlers: Vec::new(), + randomseed, + seed, + deadline, + logs, + }) + } + + fn budgeted( + &self, + budget: Duration, + f: impl FnOnce() -> mlua::Result, + ) -> Result { + self.randomseed.call::<()>(self.seed)?; + self.deadline.set(Some(Instant::now() + budget)); + let result = f(); + self.deadline.set(None); + Ok(result?) + } + + /// Run the view and parse its tree. Handler ids in the tree stay valid + /// until the next render. + pub fn render(&mut self) -> Result { + let value = self.budgeted(CALL_BUDGET, || self.view.call::(()))?; + let mut parser = node::Parser::new(); + let deadline = Instant::now() + CALL_BUDGET; + self.deadline.set(Some(deadline)); + let tree = parser.root(value); + self.deadline.set(None); + let tree = tree?; + self.handlers = parser.handlers; + Ok(tree) + } + + fn handler(&self, id: HandlerId) -> Result<&Function, VisualError> { + self.handlers + .get(id.0 as usize) + .ok_or_else(|| VisualError::script("stale handler")) + } + + /// Call an event handler. The caller renders again afterwards. + pub fn call(&mut self, id: HandlerId, arg: HandlerArg) -> Result<(), VisualError> { + let handler = self.handler(id)?.clone(); + self.budgeted(CALL_BUDGET, || match arg { + HandlerArg::None => handler.call::<()>(()), + HandlerArg::Text(text) => handler.call::<()>(text), + HandlerArg::Bool(b) => handler.call::<()>(b), + HandlerArg::Number(n) => handler.call::<()>(n), + HandlerArg::Point(x, y) => handler.call::<()>((x, y)), + }) + } + + /// Run a canvas's draw function at a size, in design pixels. + pub fn draw_canvas( + &mut self, + id: HandlerId, + width: f32, + height: f32, + time: f64, + pointer: Option, + ) -> Result, VisualError> { + let draw = self.handler(id)?.clone(); + let graphics = self + .lua + .create_userdata(draw::Graphics::new(width, height, time, pointer))?; + self.budgeted(DRAW_BUDGET, || { + draw.call::<()>((&graphics, width, height, time)) + })?; + Ok(graphics.take::()?.commands) + } + + /// Run a scene's draw function. + pub fn draw_scene(&mut self, id: HandlerId, time: f64) -> Result { + let draw = self.handler(id)?.clone(); + let scene = self.lua.create_userdata(Scene::new(time))?; + self.budgeted(DRAW_BUDGET, || draw.call::<()>((&scene, time)))?; + Ok(scene.take::()?) + } + + /// Lines the script printed, oldest first. + pub fn logs(&self) -> Vec { + self.logs.borrow().clone() + } +} + +/// Round tick values covering `lo..=hi`, about `count` of them. +pub fn nice_ticks(lo: f64, hi: f64, count: usize) -> Vec { + let (lo, hi) = if hi < lo { (hi, lo) } else { (lo, hi) }; + if !lo.is_finite() || !hi.is_finite() { + return Vec::new(); + } + if (hi - lo).abs() < f64::EPSILON { + return vec![lo]; + } + let raw = (hi - lo) / count.max(1) as f64; + let magnitude = 10f64.powf(raw.log10().floor()); + let normalized = raw / magnitude; + let step = magnitude + * if normalized < 1.5 { + 1.0 + } else if normalized < 3.0 { + 2.0 + } else if normalized < 7.0 { + 5.0 + } else { + 10.0 + }; + let mut out = Vec::new(); + let mut v = (lo / step).ceil() * step; + // Snap away float noise (0.6000000000000001) at the step's precision. + let digits = 10f64.powi((-step.log10().floor()).max(0.0) as i32); + while v <= hi + step * 1e-9 && out.len() < 100 { + out.push(((v / step).round() * step * digits).round() / digits); + v += step; + } + out +} + +/// What `show_visual` learned by running a script before showing it. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Report { + pub nodes: usize, + pub interactions: usize, + pub logs: Vec, +} + +/// Run a script the way the desktop will: setup, the first render, every +/// canvas and scene draw, and then each tab, toggle, slider end, and button +/// once with a render after each. The first failure comes back with where +/// it happened, so the agent can fix the script before the reader sees it. +pub fn check(source: &str, seed: u64) -> Result { + let mut visual = Visual::load(source, seed)?; + let tree = visual.render()?; + let mut report = Report::default(); + exercise_draws(&mut visual, &tree, &mut report)?; + + let mut interactions = Vec::new(); + collect_interactions(&tree, &mut interactions); + for (describe, path, arg) in interactions.into_iter().take(24) { + // Ids are per render; find the control again in the current tree. + let current = visual.render()?; + let Some(id) = handler_at(¤t, &path) else { + continue; + }; + visual + .call(id, arg) + .and_then(|()| { + let tree = visual.render()?; + exercise_draws(&mut visual, &tree, &mut Report::default()) + }) + .map_err(|e| VisualError::script(format!("after {describe}: {}", e.message)))?; + report.interactions += 1; + } + report.logs = visual.logs(); + Ok(report) +} + +fn exercise_draws( + visual: &mut Visual, + node: &Node, + report: &mut Report, +) -> Result<(), VisualError> { + report.nodes += 1; + match &node.kind { + NodeKind::Canvas(canvas) => { + let frames: &[f64] = if canvas.animate { &[0.0, 1.5] } else { &[0.0] }; + for time in frames { + visual + .draw_canvas( + canvas.draw, + 640.0, + canvas.height, + *time, + Some(Pointer { + x: 320.0, + y: canvas.height / 2.0, + down: false, + }), + ) + .map_err(|e| VisualError::script(format!("in a canvas draw: {}", e.message)))?; + } + } + NodeKind::Scene(scene) => { + let drawn = visual + .draw_scene(scene.draw, 0.0) + .map_err(|e| VisualError::script(format!("in a scene draw: {}", e.message)))?; + if drawn + .project( + Camera { + yaw: scene.yaw, + pitch: scene.pitch, + zoom: scene.zoom, + }, + 640.0, + scene.height, + ) + .prims + .is_empty() + { + return Err(VisualError::script( + "a scene draw added no geometry (use s:box, s:sphere, s:surface, s:mesh, \ + s:line, s:point)", + )); + } + } + _ => {} + } + for child in children(node) { + exercise_draws(visual, child, report)?; + } + Ok(()) +} + +/// Child nodes in render order, including table cells. +pub fn children(node: &Node) -> Box + '_> { + match &node.kind { + NodeKind::Stack(stack) => Box::new(stack.children.iter()), + NodeKind::Grid { children, .. } => Box::new(children.iter()), + NodeKind::Table(table) => Box::new(table.rows.iter().flatten()), + _ => Box::new(std::iter::empty()), + } +} + +type Interaction = (String, Vec, HandlerArg); + +fn collect_interactions(node: &Node, out: &mut Vec) { + fn walk(node: &Node, path: &mut Vec, out: &mut Vec) { + match &node.kind { + NodeKind::Tabs { options, value, .. } => { + for option in options.iter().filter(|o| *o != value) { + out.push(( + format!("selecting tab '{option}'"), + path.clone(), + HandlerArg::Text(option.clone()), + )); + } + out.push(( + format!("selecting tab '{value}'"), + path.clone(), + HandlerArg::Text(value.clone()), + )); + } + NodeKind::Toggle { label, value, .. } => { + out.push(( + format!("switching '{label}'"), + path.clone(), + HandlerArg::Bool(!value), + )); + } + NodeKind::Slider { + label, min, max, .. + } => { + let name = label.as_deref().unwrap_or("a slider"); + out.push(( + format!("moving {name} to its maximum"), + path.clone(), + HandlerArg::Number(*max), + )); + out.push(( + format!("moving {name} to its minimum"), + path.clone(), + HandlerArg::Number(*min), + )); + } + NodeKind::Button { label, .. } => { + out.push(( + format!("pressing '{label}'"), + path.clone(), + HandlerArg::None, + )); + } + NodeKind::Stack(Stack { + on_click: Some(_), .. + }) => { + out.push(("clicking a row".into(), path.clone(), HandlerArg::None)); + } + _ => {} + } + for (ix, child) in children(node).enumerate() { + path.push(ix); + walk(child, path, out); + path.pop(); + } + } + walk(node, &mut Vec::new(), out); +} + +fn handler_at(node: &Node, path: &[usize]) -> Option { + let mut node = node; + for ix in path { + node = children(node).nth(*ix)?; + } + match &node.kind { + NodeKind::Tabs { on_change, .. } + | NodeKind::Toggle { on_change, .. } + | NodeKind::Slider { on_change, .. } => Some(*on_change), + NodeKind::Button { on_click, .. } => Some(*on_click), + NodeKind::Stack(Stack { on_click, .. }) => *on_click, + _ => None, + } +} + +/// A stable seed from a visual's id. +pub fn seed_for(id: &str) -> u64 { + id.bytes().fold(0xcbf2_9ce4_8422_2325u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/visual/src/node.rs b/crates/visual/src/node.rs new file mode 100644 index 0000000000..1270a982c0 --- /dev/null +++ b/crates/visual/src/node.rs @@ -0,0 +1,1002 @@ +//! The element tree a visual's view function returns, parsed from the tagged +//! tables `ui.*` constructors build. The tree is plain data: the desktop +//! renders it with native elements, and handlers are indexes into the +//! render's handler list. + +use mlua::{Function, Table, Value}; + +use crate::VisualError; +use crate::style::{Align, Color, Justify, TextSize, Token, Weight}; + +/// A Lua function a node calls back (an event handler or a draw function), +/// valid until the next render. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HandlerId(pub u32); + +#[derive(Debug, Clone, PartialEq)] +pub struct Node { + pub kind: NodeKind, + /// Fixed width in design pixels, inside a row or grid. + pub width: Option, + /// Takes the row's spare width. + pub grow: bool, + /// The script's stable name for this node, so view-local state (a + /// chart's hover, a scene's camera) survives the tree changing shape. + pub key: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Axis { + #[default] + Vertical, + Horizontal, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum NodeKind { + Stack(Stack), + Grid { + columns: u16, + gap: f32, + children: Vec, + }, + Text(Text), + /// A hairline divider. + Rule, + Space(f32), + /// A headline figure over a short label. + Stat { + value: String, + label: String, + /// A short change note beside the figure ("+12%"). + delta: Option, + tone: Option, + }, + Table(Table_), + Chart(Box), + Canvas(Canvas), + Scene(SceneNode), + Button { + label: String, + on_click: HandlerId, + primary: bool, + }, + Tabs { + options: Vec, + value: String, + on_change: HandlerId, + }, + Toggle { + label: String, + value: bool, + on_change: HandlerId, + }, + Slider { + label: Option, + value: f64, + min: f64, + max: f64, + step: f64, + on_change: HandlerId, + format: NumberFormat, + }, + Link { + label: String, + url: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Stack { + pub axis: Axis, + pub gap: f32, + pub align: Align, + pub justify: Justify, + pub wrap: bool, + /// The hover wash, the only fill a visual has: for rows the reader can + /// click or scan. + pub hover: bool, + pub on_click: Option, + pub children: Vec, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Text { + pub spans: Vec, + pub size: TextSize, + pub weight: Weight, + pub color: Option, + pub mono: bool, + pub italic: bool, + pub align: Align, + pub truncate: bool, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Span { + pub text: String, + pub weight: Option, + pub color: Option, + pub mono: bool, + pub italic: bool, +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Table_ { + pub columns: Vec, + pub rows: Vec>, + /// Rows take the hover wash. + pub hover: bool, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Column { + pub label: String, + pub align: Align, + pub width: Option, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct NumberFormat { + pub prefix: String, + pub unit: String, + pub decimals: Option, +} + +impl NumberFormat { + pub fn format(&self, value: f64) -> String { + let decimals = self.decimals.map(usize::from).unwrap_or_else(|| { + let magnitude = value.abs(); + if magnitude == 0.0 || magnitude >= 100.0 || value.fract() == 0.0 { + 0 + } else if magnitude >= 10.0 { + 1 + } else { + 2 + } + }); + let text = format!("{value:.decimals$}"); + let (sign, digits) = match text.strip_prefix('-') { + Some(rest) => ("-", rest), + None => ("", text.as_str()), + }; + let (int, frac) = digits.split_once('.').unwrap_or((digits, "")); + let mut grouped = String::new(); + for (ix, ch) in int.chars().enumerate() { + if ix > 0 && (int.len() - ix) % 3 == 0 { + grouped.push(','); + } + grouped.push(ch); + } + let frac = if frac.is_empty() { + String::new() + } else { + format!(".{frac}") + }; + let space = if self.unit.is_empty() || self.unit.starts_with('%') { + "" + } else { + " " + }; + format!("{sign}{}{grouped}{frac}{space}{}", self.prefix, self.unit) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChartKind { + Bar, + Line, + Area, + Scatter, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Chart { + pub kind: ChartKind, + pub height: f32, + /// Category labels (bar, line, area), one per value. + pub x: Vec, + pub series: Vec, + pub stacked: bool, + pub horizontal: bool, + pub y_min: Option, + pub y_max: Option, + pub format: NumberFormat, + /// Scatter x axis format. + pub x_format: NumberFormat, + pub legend: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Series { + pub name: String, + pub color: Color, + /// One value per category; `nil` leaves a gap. + pub values: Vec>, + /// Scatter points. + pub points: Vec<(f64, f64)>, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Canvas { + pub height: f32, + pub draw: HandlerId, + pub animate: bool, + pub on_click: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SceneNode { + pub height: f32, + pub draw: HandlerId, + pub animate: bool, + pub orbit: bool, + /// Turntable speed in radians per second while the reader isn't dragging. + pub spin: f32, + pub yaw: f32, + pub pitch: f32, + pub zoom: f32, +} + +pub const MIN_HEIGHT: f32 = 40.0; +pub const MAX_HEIGHT: f32 = 720.0; +const MAX_NODES: usize = 4000; +const MAX_DEPTH: usize = 48; +const MAX_TEXT: usize = 8000; + +/// Builds a [`Node`] tree from a view's return value, collecting the +/// functions it references. +pub(crate) struct Parser { + pub handlers: Vec, + nodes: usize, +} + +/// Field names every node accepts. +const COMMON: [&str; 4] = ["__kind", "width", "grow", "key"]; + +/// Props that exist in web styling but not here, with the reason, so the +/// agent learns the rule from the error. +fn forbidden(prop: &str) -> Option<&'static str> { + Some(match prop { + "background" | "bg" | "fill" | "shadow" | "border" | "padding" | "radius" | "blur" + | "opacity" => { + "visuals have no containers, fills, borders, or shadows: lay content directly on \ + the chat with gap, ui.rule(), and type" + } + "font" | "family" | "font_family" | "fontFamily" | "font_size" | "fontSize" => { + "text always renders in Zeron's interface font; size it with size = xs, sm, body, \ + lg, xl, or 2xl" + } + _ => return None, + }) +} + +impl Parser { + pub fn new() -> Self { + Self { + handlers: Vec::new(), + nodes: 0, + } + } + + fn handler(&mut self, function: Function) -> HandlerId { + self.handlers.push(function); + HandlerId(self.handlers.len() as u32 - 1) + } + + pub fn root(&mut self, value: Value) -> Result { + let mut children = Vec::new(); + self.children_into(value, &mut children, 0)?; + Ok(match children.len() { + 1 => children.pop().unwrap(), + _ => Node { + kind: NodeKind::Stack(Stack { + gap: 16.0, + children, + ..Default::default() + }), + width: None, + grow: false, + key: None, + }, + }) + } + + /// Children may be nodes, strings (plain text), nested lists of either + /// (from a map), or nil/false (conditional rendering). + fn children_into( + &mut self, + value: Value, + out: &mut Vec, + depth: usize, + ) -> Result<(), VisualError> { + if depth > MAX_DEPTH { + return Err(VisualError::script("the view nests too deeply")); + } + match value { + Value::Nil | Value::Boolean(false) => {} + Value::String(s) => out.push(self.plain_text(s.to_string_lossy())?), + Value::Integer(_) | Value::Number(_) => out.push(self.plain_text(number_text(&value))?), + Value::Table(table) => { + if table.raw_get::>("__kind")?.is_some() { + out.push(self.node(&table, depth)?); + } else { + for child in table.sequence_values::() { + self.children_into(child?, out, depth + 1)?; + } + } + } + other => { + return Err(VisualError::script(format!( + "a view child must be a ui node or a string, not a {}", + other.type_name() + ))); + } + } + Ok(()) + } + + fn count(&mut self) -> Result<(), VisualError> { + self.nodes += 1; + if self.nodes > MAX_NODES { + return Err(VisualError::script(format!( + "the view has more than {MAX_NODES} nodes; summarize or paginate the data" + ))); + } + Ok(()) + } + + fn plain_text(&mut self, text: String) -> Result { + self.count()?; + Ok(Node { + kind: NodeKind::Text(Text { + spans: vec![Span { + text: clip(text), + ..Default::default() + }], + ..Default::default() + }), + width: None, + grow: false, + key: None, + }) + } + + fn node(&mut self, table: &Table, depth: usize) -> Result { + self.count()?; + let kind: String = table.raw_get("__kind")?; + let props = Props { table, kind: &kind }; + let node_kind = match kind.as_str() { + "column" | "row" => { + props.allow(&["gap", "align", "justify", "wrap", "hover", "on_click"])?; + let mut children = Vec::new(); + for child in table.sequence_values::() { + self.children_into(child?, &mut children, depth + 1)?; + } + NodeKind::Stack(Stack { + axis: if kind == "row" { + Axis::Horizontal + } else { + Axis::Vertical + }, + gap: props + .number("gap")? + .unwrap_or(if kind == "row" { 12.0 } else { 8.0 }) + .clamp(0.0, 96.0) as f32, + align: props.parsed("align", Align::parse)?.unwrap_or_default(), + justify: props.parsed("justify", Justify::parse)?.unwrap_or_default(), + wrap: props.bool("wrap")?.unwrap_or(false), + hover: props.bool("hover")?.unwrap_or(false), + on_click: props.function("on_click")?.map(|f| self.handler(f)), + children, + }) + } + "grid" => { + props.allow(&["columns", "gap"])?; + let mut children = Vec::new(); + for child in table.sequence_values::() { + self.children_into(child?, &mut children, depth + 1)?; + } + NodeKind::Grid { + columns: props.number("columns")?.unwrap_or(3.0).clamp(1.0, 12.0) as u16, + gap: props.number("gap")?.unwrap_or(16.0).clamp(0.0, 96.0) as f32, + children, + } + } + "text" => NodeKind::Text(self.text(&props)?), + "rule" => { + props.allow(&[])?; + NodeKind::Rule + } + "space" => { + props.allow(&["size"])?; + NodeKind::Space(props.number("size")?.unwrap_or(16.0).clamp(0.0, 200.0) as f32) + } + "stat" => { + props.allow(&["value", "label", "delta", "tone"])?; + NodeKind::Stat { + value: props.text_value("value")?.unwrap_or_default(), + label: props.string("label")?.unwrap_or_default(), + delta: props.text_value("delta")?, + tone: props.color("tone")?, + } + } + "table" => NodeKind::Table(self.table(&props, depth)?), + "chart" => NodeKind::Chart(Box::new(chart(&props)?)), + "canvas" => { + props.allow(&["height", "draw", "animate", "on_click"])?; + NodeKind::Canvas(Canvas { + height: props.height()?, + draw: self.required_function(&props, "draw")?, + animate: props.bool("animate")?.unwrap_or(false), + on_click: props.function("on_click")?.map(|f| self.handler(f)), + }) + } + "scene" => { + props.allow(&["height", "draw", "animate", "orbit", "spin", "camera"])?; + let camera: Option
= props.get("camera")?; + let (mut yaw, mut pitch, mut zoom) = (0.6, 0.45, 1.0); + if let Some(camera) = camera { + yaw = camera.get::>("yaw")?.unwrap_or(yaw); + pitch = camera.get::>("pitch")?.unwrap_or(pitch); + zoom = camera.get::>("zoom")?.unwrap_or(zoom); + } + NodeKind::Scene(SceneNode { + height: props.height()?, + draw: self.required_function(&props, "draw")?, + animate: props.bool("animate")?.unwrap_or(false), + orbit: props.bool("orbit")?.unwrap_or(true), + spin: props.number("spin")?.unwrap_or(0.0).clamp(-3.0, 3.0) as f32, + yaw: yaw as f32, + pitch: (pitch as f32).clamp(-1.5, 1.5), + zoom: (zoom as f32).clamp(0.2, 5.0), + }) + } + "button" => { + props.allow(&["label", "on_click", "primary"])?; + NodeKind::Button { + label: props.required_string("label")?, + on_click: self.required_function(&props, "on_click")?, + primary: props.bool("primary")?.unwrap_or(false), + } + } + "tabs" => { + props.allow(&["options", "value", "on_change"])?; + let options: Vec = props + .get::>>("options")? + .unwrap_or_default() + .iter() + .map(value_text) + .collect(); + if options.is_empty() { + return Err(props.error("needs options = { ... }")); + } + let value = props + .text_value("value")? + .unwrap_or_else(|| options[0].clone()); + NodeKind::Tabs { + options, + value, + on_change: self.required_function(&props, "on_change")?, + } + } + "toggle" => { + props.allow(&["label", "value", "on_change"])?; + NodeKind::Toggle { + label: props.string("label")?.unwrap_or_default(), + value: props.bool("value")?.unwrap_or(false), + on_change: self.required_function(&props, "on_change")?, + } + } + "slider" => { + props.allow(&[ + "label", + "value", + "min", + "max", + "step", + "on_change", + "unit", + "prefix", + "decimals", + ])?; + let min = props.number("min")?.unwrap_or(0.0); + let max = props.number("max")?.unwrap_or(1.0).max(min + f64::EPSILON); + NodeKind::Slider { + label: props.string("label")?, + value: props.number("value")?.unwrap_or(min).clamp(min, max), + min, + max, + step: props.number("step")?.unwrap_or(0.0).max(0.0), + on_change: self.required_function(&props, "on_change")?, + format: props.format()?, + } + } + "link" => { + props.allow(&["label", "url"])?; + let url = props.required_string("url")?; + if !(url.starts_with("https://") || url.starts_with("http://")) { + return Err(props.error("url must be http(s)")); + } + NodeKind::Link { + label: props.string("label")?.unwrap_or_else(|| url.clone()), + url, + } + } + "span" => { + return Err(VisualError::script( + "ui.span goes inside ui.text { ... }, not directly in a layout", + )); + } + other => return Err(VisualError::script(format!("unknown node '{other}'"))), + }; + Ok(Node { + kind: node_kind, + width: props.number("width")?.map(|w| w.clamp(0.0, 2000.0) as f32), + grow: props.bool("grow")?.unwrap_or(false), + key: props.string("key")?, + }) + } + + fn required_function(&mut self, props: &Props, name: &str) -> Result { + match props.function(name)? { + Some(function) => Ok(self.handler(function)), + None => Err(props.error(&format!("needs {name} = function(...) end"))), + } + } + + fn text(&mut self, props: &Props) -> Result { + props.allow(&[ + "size", "weight", "color", "mono", "italic", "align", "truncate", + ])?; + let mut spans = Vec::new(); + let mut len = 0usize; + for child in props.table.sequence_values::() { + let span = match child? { + Value::Table(table) + if table.raw_get::>("__kind")?.as_deref() == Some("span") => + { + let span_props = Props { + table: &table, + kind: "span", + }; + span_props.allow(&["weight", "color", "mono", "italic"])?; + let text: String = table + .sequence_values::() + .map(|v| v.map(|v| value_text(&v))) + .collect::, _>>()? + .concat(); + Span { + text, + weight: span_props.weight()?, + color: span_props.color("color")?, + mono: span_props.bool("mono")?.unwrap_or(false), + italic: span_props.bool("italic")?.unwrap_or(false), + } + } + Value::Table(_) => { + return Err(props.error("children must be strings or ui.span { ... }")); + } + Value::Nil | Value::Boolean(false) => continue, + value => Span { + text: value_text(&value), + ..Default::default() + }, + }; + len += span.text.len(); + spans.push(span); + } + if len > MAX_TEXT { + return Err(props.error(&format!("is over {MAX_TEXT} characters"))); + } + Ok(Text { + spans, + size: props.parsed("size", TextSize::parse)?.unwrap_or_default(), + weight: props.weight()?.unwrap_or_default(), + color: props.color("color")?, + mono: props.bool("mono")?.unwrap_or(false), + italic: props.bool("italic")?.unwrap_or(false), + align: props.parsed("align", Align::parse)?.unwrap_or_default(), + truncate: props.bool("truncate")?.unwrap_or(false), + }) + } + + fn table(&mut self, props: &Props, depth: usize) -> Result { + props.allow(&["columns", "rows", "hover"])?; + let mut columns = Vec::new(); + for column in props + .get::>>("columns")? + .unwrap_or_default() + { + columns.push(match column { + Value::Table(t) => Column { + label: t.get::>("label")?.unwrap_or_default(), + align: match t.get::>("align")? { + Some(a) => Align::parse(&a)?, + None => Align::Start, + }, + width: t.get::>("width")?.map(|w| w as f32), + }, + other => Column { + label: value_text(&other), + ..Default::default() + }, + }); + } + let mut rows = Vec::new(); + for row in props.get::>>("rows")?.unwrap_or_default() { + let mut cells = Vec::new(); + for cell in row.sequence_values::() { + let cell = cell?; + let numeric = matches!(cell, Value::Integer(_) | Value::Number(_)); + let mut out = Vec::new(); + self.children_into(cell, &mut out, depth + 1)?; + let node = match out.len() { + 0 => self.plain_text(String::new())?, + 1 => out.pop().unwrap(), + _ => Node { + kind: NodeKind::Stack(Stack { + axis: Axis::Horizontal, + gap: 6.0, + children: out, + ..Default::default() + }), + width: None, + grow: false, + key: None, + }, + }; + // Numbers right-align when the script didn't say otherwise. + if numeric + && let Some(column) = columns.get_mut(cells.len()) + && column.align == Align::Start + { + column.align = Align::End; + } + cells.push(node); + } + rows.push(cells); + } + if rows.len() > 500 { + return Err(props.error("has more than 500 rows; summarize or paginate")); + } + Ok(Table_ { + columns, + rows, + hover: props.bool("hover")?.unwrap_or(true), + }) + } +} + +fn chart(props: &Props) -> Result { + props.allow(&[ + "type", + "height", + "x", + "series", + "stacked", + "horizontal", + "min", + "max", + "unit", + "prefix", + "decimals", + "x_unit", + "x_decimals", + "legend", + ])?; + let kind = match props.string("type")?.as_deref().unwrap_or("bar") { + "bar" => ChartKind::Bar, + "line" => ChartKind::Line, + "area" => ChartKind::Area, + "scatter" => ChartKind::Scatter, + other => { + return Err(props.error(&format!( + "has unknown type '{other}': use bar, line, area, or scatter (draw anything else \ + with ui.canvas)" + ))); + } + }; + let x: Vec = props + .get::>>("x")? + .unwrap_or_default() + .iter() + .map(value_text) + .collect(); + let mut series = Vec::new(); + for (ix, s) in props + .get::>>("series")? + .unwrap_or_default() + .into_iter() + .enumerate() + { + let s_props = Props { + table: &s, + kind: "chart series", + }; + s_props.allow(&["name", "values", "points", "color"])?; + let values = match s.get::>("values")? { + Some(values) => { + let len = values.raw_len(); + (1..=len) + .map(|i| values.raw_get::>(i)) + .collect::, _>>()? + } + None => Vec::new(), + }; + let mut points = Vec::new(); + for point in s.get::>>("points")?.unwrap_or_default() { + let px: f64 = point.raw_get(1).or_else(|_| point.get("x"))?; + let py: f64 = point.raw_get(2).or_else(|_| point.get("y"))?; + points.push((px, py)); + } + if values.len() + points.len() > 20_000 { + return Err(props.error("has more than 20,000 values in a series")); + } + series.push(Series { + name: s_props.string("name")?.unwrap_or_default(), + color: s_props + .color("color")? + .unwrap_or(Color::token(Token::series(ix))), + values, + points, + }); + } + if series.is_empty() { + return Err(props.error("needs series = { { name = ..., values = { ... } } }")); + } + if kind == ChartKind::Scatter && series.iter().all(|s| s.points.is_empty()) { + return Err(props.error("of type scatter needs series with points = { {x, y}, ... }")); + } + if kind != ChartKind::Scatter { + let longest = series.iter().map(|s| s.values.len()).max().unwrap_or(0); + if longest == 0 { + return Err(props.error("needs values in its series")); + } + if x.len() > 2000 || longest > 2000 { + return Err(props.error("has more than 2,000 categories")); + } + } + let format = props.format()?; + Ok(Chart { + kind, + height: props.number("height")?.unwrap_or(220.0).clamp(80.0, 600.0) as f32, + x, + series, + stacked: props.bool("stacked")?.unwrap_or(false), + horizontal: props.bool("horizontal")?.unwrap_or(false), + y_min: props.number("min")?, + y_max: props.number("max")?, + format, + x_format: NumberFormat { + prefix: String::new(), + unit: props.string("x_unit")?.unwrap_or_default(), + decimals: props.number("x_decimals")?.map(|d| d.clamp(0.0, 6.0) as u8), + }, + legend: props.bool("legend")?.unwrap_or(true), + }) +} + +struct Props<'a> { + table: &'a Table, + kind: &'a str, +} + +impl Props<'_> { + fn error(&self, message: &str) -> VisualError { + VisualError::script(format!("ui.{} {message}", self.kind)) + } + + /// Reject props the node doesn't know, so a typo or a web-style prop + /// fails at `show_visual` time instead of being silently ignored. + fn allow(&self, known: &[&str]) -> Result<(), VisualError> { + for pair in self.table.pairs::() { + let (key, _) = pair?; + let Value::String(key) = key else { continue }; + let key = key.to_string_lossy(); + if COMMON.contains(&key.as_str()) || known.contains(&key.as_str()) { + continue; + } + let reason = forbidden(&key) + .map(|r| format!(": {r}")) + .unwrap_or_else(|| { + if known.is_empty() { + String::new() + } else { + format!(" (it takes {})", known.join(", ")) + } + }); + return Err(self.error(&format!("has no '{key}'{reason}"))); + } + Ok(()) + } + + fn get(&self, name: &str) -> Result { + self.table + .raw_get(name) + .map_err(|e| self.error(&format!("'{name}': {}", short_error(&e)))) + } + + fn number(&self, name: &str) -> Result, VisualError> { + self.get(name) + } + + fn bool(&self, name: &str) -> Result, VisualError> { + match self.table.raw_get::(name)? { + Value::Nil => Ok(None), + Value::Boolean(b) => Ok(Some(b)), + _ => Err(self.error(&format!("'{name}' must be true or false"))), + } + } + + fn string(&self, name: &str) -> Result, VisualError> { + Ok(match self.table.raw_get::(name)? { + Value::Nil => None, + value @ (Value::String(_) | Value::Integer(_) | Value::Number(_)) => { + Some(clip(value_text(&value))) + } + _ => return Err(self.error(&format!("'{name}' must be a string"))), + }) + } + + fn required_string(&self, name: &str) -> Result { + self.string(name)? + .ok_or_else(|| self.error(&format!("needs {name}"))) + } + + /// A string or a number (formatted as Lua prints it). + fn text_value(&self, name: &str) -> Result, VisualError> { + self.string(name) + } + + fn function(&self, name: &str) -> Result, VisualError> { + match self.table.raw_get::(name)? { + Value::Nil => Ok(None), + Value::Function(f) => Ok(Some(f)), + _ => Err(self.error(&format!("'{name}' must be a function"))), + } + } + + fn parsed( + &self, + name: &str, + parse: fn(&str) -> Result, + ) -> Result, VisualError> { + self.string(name)?.map(|s| parse(&s)).transpose() + } + + fn weight(&self) -> Result, VisualError> { + Ok(match self.table.raw_get::("weight")? { + Value::Nil => None, + Value::Integer(n) => Some(Weight::from_number(n as f64)), + Value::Number(n) => Some(Weight::from_number(n)), + Value::String(s) => Some(Weight::parse(&s.to_string_lossy())?), + _ => return Err(self.error("'weight' must be a number or a name")), + }) + } + + fn color(&self, name: &str) -> Result, VisualError> { + parse_color(self.table.raw_get::(name)?).map_err(|e| self.error(&e.to_string())) + } + + fn height(&self) -> Result { + match self.number("height")? { + Some(h) => Ok((h as f32).clamp(MIN_HEIGHT, MAX_HEIGHT)), + None => Err(self.error(&format!( + "needs height (in pixels, {MIN_HEIGHT}-{MAX_HEIGHT}); its width is the reply \ + column's" + ))), + } + } + + fn format(&self) -> Result { + Ok(NumberFormat { + prefix: self.string("prefix")?.unwrap_or_default(), + unit: self.string("unit")?.unwrap_or_default(), + decimals: self.number("decimals")?.map(|d| d.clamp(0.0, 6.0) as u8), + }) + } +} + +/// A color: a token name, or a `ui.color` / `ui.mix` table. +pub(crate) fn parse_color(value: Value) -> Result, VisualError> { + Ok(match value { + Value::Nil => None, + Value::String(s) => Some(Color::token(Token::parse(&s.to_string_lossy())?)), + Value::Table(t) if t.raw_get::>("__color")?.unwrap_or(false) => { + let token = Token::parse(&t.raw_get::("token")?)?; + let mix = match t.raw_get::>("mix")? { + Some(other) => Some(( + Token::parse(&other)?, + t.raw_get::>("t")? + .unwrap_or(0.5) + .clamp(0.0, 1.0) as f32, + )), + None => None, + }; + Some(Color { + token, + mix, + alpha: t + .raw_get::>("alpha")? + .unwrap_or(1.0) + .clamp(0.0, 1.0) as f32, + }) + } + other => { + return Err(VisualError::script(format!( + "a color is a theme token name or ui.color/ui.mix, not a {}", + other.type_name() + ))); + } + }) +} + +pub(crate) fn value_text(value: &Value) -> String { + match value { + Value::String(s) => s.to_string_lossy(), + Value::Integer(_) | Value::Number(_) => number_text(value), + Value::Boolean(b) => b.to_string(), + Value::Nil => String::new(), + other => other.type_name().to_owned(), + } +} + +fn number_text(value: &Value) -> String { + match value { + Value::Integer(n) => n.to_string(), + Value::Number(n) if n.fract() == 0.0 && n.abs() < 1e15 => format!("{}", *n as i64), + Value::Number(n) => { + let text = format!("{n:.4}"); + text.trim_end_matches('0').trim_end_matches('.').to_owned() + } + _ => String::new(), + } +} + +fn clip(mut text: String) -> String { + if text.len() > MAX_TEXT { + let mut end = MAX_TEXT; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + text +} + +pub(crate) fn short_error(error: &mlua::Error) -> String { + match error { + mlua::Error::FromLuaConversionError { from, to, .. } => { + format!("expected {to}, got {from}") + } + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn numbers_format_with_grouping_and_units() { + let f = NumberFormat { + unit: "ms".into(), + ..Default::default() + }; + assert_eq!(f.format(1234.0), "1,234 ms"); + assert_eq!(f.format(-12.5), "-12.5 ms"); + let money = NumberFormat { + prefix: "$".into(), + decimals: Some(2), + ..Default::default() + }; + assert_eq!(money.format(1234567.891), "$1,234,567.89"); + let pct = NumberFormat { + unit: "%".into(), + ..Default::default() + }; + assert_eq!(pct.format(4.256), "4.26%"); + } +} diff --git a/crates/visual/src/prelude.luau b/crates/visual/src/prelude.luau new file mode 100644 index 0000000000..6cf9f12fc6 --- /dev/null +++ b/crates/visual/src/prelude.luau @@ -0,0 +1,99 @@ +-- The `ui` library every visual script sees. Constructors only tag their +-- props table; Zeron validates and renders the tree natively. +local ui = {} + +local function constructor(kind) + return function(props) + if type(props) ~= "table" or rawget(props, "__kind") ~= nil then + props = { props } + end + props.__kind = kind + return props + end +end + +for _, kind in + { + "column", + "row", + "grid", + "text", + "span", + "rule", + "space", + "stat", + "table", + "chart", + "canvas", + "scene", + "button", + "tabs", + "toggle", + "slider", + "link", + } +do + ui[kind] = constructor(kind) +end + +-- A theme token at an opacity. +function ui.color(token, alpha) + return { __color = true, token = token, alpha = alpha } +end + +-- A blend of two theme tokens, `t` of the way from `a` to `b`. +function ui.mix(a, b, t) + return { __color = true, token = a, mix = b, t = t } +end + +-- The `i`th categorical series color (1-based, cycling through six). +function ui.series(i) + return "chart" .. ((i - 1) % 6 + 1) +end + +-- The table your handlers change. Zeron renders the view again after every +-- handler, so plain assignment is enough. +function ui.state(initial) + return initial or {} +end + +-- A linear map from the domain [d0, d1] onto the range [r0, r1]. +function ui.scale(d0, d1, r0, r1) + local span = d1 - d0 + if span == 0 then + span = 1 + end + return function(v) + return r0 + (v - d0) / span * (r1 - r0) + end +end + +function ui.clamp(v, lo, hi) + return math.max(lo, math.min(hi, v)) +end + +function ui.lerp(a, b, t) + return a + (b - a) * t +end + +function ui.map(list, f) + local out = {} + for i, v in ipairs(list) do + out[i] = f(v, i) + end + return out +end + +function ui.range(a, b, step) + if b == nil then + a, b = 1, a + end + step = step or 1 + local out = {} + for v = a, b, step do + table.insert(out, v) + end + return out +end + +return ui diff --git a/crates/visual/src/scene.rs b/crates/visual/src/scene.rs new file mode 100644 index 0000000000..01e6f5b95a --- /dev/null +++ b/crates/visual/src/scene.rs @@ -0,0 +1,982 @@ +//! `ui.scene` 3D: the draw function gets a scene object whose methods add +//! world-space geometry (y up). Zeron frames the camera on the geometry, +//! orbits it on drag, shades faces with one soft light, and projects +//! everything to depth-sorted 2D polygons that the desktop paints like any +//! other vector shape, so a 3D view stays transparent over glass and uses the +//! theme's colors. + +use mlua::{Function, Table, UserData, UserDataMethods, Value}; + +use crate::draw::lua_error; +use crate::node::{parse_color, value_text}; +use crate::style::{Color, TextSize, Token}; + +pub const MAX_FACES: usize = 30_000; +const MAX_ITEMS: usize = 20_000; + +pub type V3 = [f32; 3]; + +#[derive(Debug, Clone, PartialEq)] +struct Face { + v: [V3; 4], + /// 3 or 4 corners. + n: u8, + normal: V3, + color: Color, + /// Closed solids cull faces turned away from the camera; open surfaces + /// and meshes light both sides. + two_sided: bool, +} + +#[derive(Debug, Clone, PartialEq)] +struct Line3 { + a: V3, + b: V3, + color: Color, + width: f32, +} + +#[derive(Debug, Clone, PartialEq)] +struct Point3 { + at: V3, + radius: f32, + color: Color, + hint: Option, +} + +#[derive(Debug, Clone, PartialEq)] +struct Label3 { + at: V3, + text: String, + color: Color, + size: TextSize, +} + +/// The geometry one draw call produced. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Scene { + faces: Vec, + lines: Vec, + points: Vec, + labels: Vec, + pub time: f64, +} + +/// The orbit camera: angles in radians around the geometry's center. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Camera { + pub yaw: f32, + pub pitch: f32, + pub zoom: f32, +} + +/// One painted primitive, in canvas pixels. +#[derive(Debug, Clone, PartialEq)] +pub enum Prim { + Face { + points: Vec<[f32; 2]>, + color: Color, + /// Lighting, 0 (unlit) to 1 (facing the light). + shade: f32, + }, + Line { + a: [f32; 2], + b: [f32; 2], + color: Color, + width: f32, + }, + Dot { + at: [f32; 2], + radius: f32, + color: Color, + }, + Label { + at: [f32; 2], + text: String, + color: Color, + size: TextSize, + }, +} + +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Projection { + /// Back to front; labels last. + pub prims: Vec, + /// Hover hints: screen position, radius, text. + pub hints: Vec<([f32; 2], f32, String)>, +} + +fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +fn scale(a: V3, s: f32) -> V3 { + [a[0] * s, a[1] * s, a[2] * s] +} +fn dot(a: V3, b: V3) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +fn cross(a: V3, b: V3) -> V3 { + [ + a[1] * b[2] - a[2] * b[1], + a[2] * b[0] - a[0] * b[2], + a[0] * b[1] - a[1] * b[0], + ] +} +fn normalize(a: V3) -> V3 { + let len = dot(a, a).sqrt(); + if len < 1e-9 { + [0.0, 1.0, 0.0] + } else { + scale(a, 1.0 / len) + } +} + +impl Scene { + pub fn new(time: f64) -> Self { + Self { + time, + ..Default::default() + } + } + + fn face( + &mut self, + corners: &[V3], + color: Color, + outward: Option, + two_sided: bool, + ) -> mlua::Result<()> { + if self.faces.len() >= MAX_FACES { + return Err(lua_error(format!( + "a scene draws at most {MAX_FACES} faces" + ))); + } + if corners.iter().flatten().any(|c| !c.is_finite()) { + return Err(lua_error("a scene vertex is NaN or infinite")); + } + let mut normal = normalize(cross( + sub(corners[1], corners[0]), + sub(corners[corners.len() - 1], corners[0]), + )); + if corners.len() == 4 { + // A non-planar quad: average both diagonals' normals. + normal = normalize(cross( + sub(corners[2], corners[0]), + sub(corners[3], corners[1]), + )); + } + if let Some(outward) = outward + && dot(normal, outward) < 0.0 + { + normal = scale(normal, -1.0); + } + let mut v = [[0.0; 3]; 4]; + v[..corners.len()].copy_from_slice(corners); + self.faces.push(Face { + v, + n: corners.len() as u8, + normal, + color, + two_sided, + }); + Ok(()) + } + + fn polygon( + &mut self, + corners: &[V3], + color: Color, + outward: Option, + two_sided: bool, + ) -> mlua::Result<()> { + match corners.len() { + 0..=2 => Ok(()), + 3 | 4 => self.face(corners, color, outward, two_sided), + _ => { + for i in 1..corners.len() - 1 { + self.face( + &[corners[0], corners[i], corners[i + 1]], + color, + outward, + two_sided, + )?; + } + Ok(()) + } + } + } + + fn items(&self) -> usize { + self.lines.len() + self.points.len() + self.labels.len() + } + + fn check_items(&self) -> mlua::Result<()> { + if self.items() >= MAX_ITEMS { + return Err(lua_error(format!( + "a scene draws at most {MAX_ITEMS} lines, points, and labels" + ))); + } + Ok(()) + } + + pub fn face_count(&self) -> usize { + self.faces.len() + } + + fn bounds(&self) -> Option<(V3, V3)> { + let mut lo = [f32::INFINITY; 3]; + let mut hi = [f32::NEG_INFINITY; 3]; + let mut grow = |p: V3| { + for i in 0..3 { + lo[i] = lo[i].min(p[i]); + hi[i] = hi[i].max(p[i]); + } + }; + for face in &self.faces { + for v in &face.v[..face.n as usize] { + grow(*v); + } + } + for line in &self.lines { + grow(line.a); + grow(line.b); + } + for point in &self.points { + grow(point.at); + } + for label in &self.labels { + grow(label.at); + } + lo[0].is_finite().then_some((lo, hi)) + } + + /// Frame the geometry, orbit to `camera`, light, cull, and depth-sort. + pub fn project(&self, camera: Camera, width: f32, height: f32) -> Projection { + let Some((lo, hi)) = self.bounds() else { + return Projection::default(); + }; + let target = scale(add(lo, hi), 0.5); + let half = scale(sub(hi, lo), 0.5); + let radius = dot(half, half).sqrt().max(1e-3); + let pitch = camera.pitch.clamp(-1.52, 1.52); + // Fit the geometry's extent as the turntable sees it: the footprint's + // radius sideways, and its height plus the tilted footprint + // vertically. Neither depends on yaw, so a spinning scene keeps its + // size. + let footprint = half[0].hypot(half[2]).max(1e-3); + let tall = (half[1] * pitch.cos().abs() + footprint * pitch.sin().abs()).max(1e-3); + let distance = radius * 4.0; + let eye = add( + target, + scale( + [ + pitch.cos() * camera.yaw.sin(), + pitch.sin(), + pitch.cos() * camera.yaw.cos(), + ], + distance, + ), + ); + let forward = normalize(sub(target, eye)); + let right = normalize(cross(forward, [0.0, 1.0, 0.0])); + let up = cross(right, forward); + let focal = 0.86 + * (width / 2.0 / footprint).min(height / 2.0 / tall) + * distance + * camera.zoom.max(0.05); + let near = distance * 0.02; + let (cx, cy) = (width / 2.0, height / 2.0); + let view = |p: V3| { + let d = sub(p, eye); + [dot(d, right), dot(d, up), dot(d, forward)] + }; + let screen = |v: V3| [cx + focal * v[0] / v[2], cy - focal * v[1] / v[2]]; + // A soft key light from the upper left, in front of the geometry. + let light = normalize(add( + add(scale(right, -0.45), scale(up, 0.8)), + scale(forward, -0.55), + )); + + let mut sorted: Vec<(f32, Prim)> = Vec::with_capacity(self.faces.len() + self.items()); + for face in &self.faces { + let corners = &face.v[..face.n as usize]; + let viewed: Vec = corners.iter().map(|v| view(*v)).collect(); + if viewed.iter().any(|v| v[2] < near) { + continue; + } + let centroid = scale( + corners.iter().fold([0.0; 3], |acc, v| add(acc, *v)), + 1.0 / corners.len() as f32, + ); + let toward_eye = dot(face.normal, sub(eye, centroid)); + if !face.two_sided && toward_eye < 0.0 { + continue; + } + let mut normal = face.normal; + if toward_eye < 0.0 { + normal = scale(normal, -1.0); + } + let lambert = dot(normal, light).max(0.0); + let depth = viewed.iter().map(|v| v[2]).sum::() / viewed.len() as f32; + sorted.push(( + depth, + Prim::Face { + points: viewed.iter().map(|v| screen(*v)).collect(), + color: face.color, + shade: 0.35 + 0.65 * lambert, + }, + )); + } + for line in &self.lines { + let (a, b) = (view(line.a), view(line.b)); + if a[2] < near || b[2] < near { + continue; + } + sorted.push(( + (a[2] + b[2]) / 2.0 - radius * 1e-3, + Prim::Line { + a: screen(a), + b: screen(b), + color: line.color, + width: line.width, + }, + )); + } + let mut hints = Vec::new(); + for point in &self.points { + let v = view(point.at); + if v[2] < near { + continue; + } + let at = screen(v); + if let Some(hint) = &point.hint { + hints.push((at, point.radius.max(4.0), hint.clone())); + } + sorted.push(( + v[2] - radius * 2e-3, + Prim::Dot { + at, + radius: point.radius, + color: point.color, + }, + )); + } + sorted.sort_by(|a, b| b.0.total_cmp(&a.0)); + let mut prims: Vec = sorted.into_iter().map(|(_, p)| p).collect(); + for label in &self.labels { + let v = view(label.at); + if v[2] < near { + continue; + } + prims.push(Prim::Label { + at: screen(v), + text: label.text.clone(), + color: label.color, + size: label.size, + }); + } + Projection { prims, hints } + } +} + +fn v3(value: Value, what: &str) -> mlua::Result { + match value { + Value::Table(t) => { + let get = |i: usize, key: &str| -> mlua::Result { + match t.raw_get::>(i)? { + Some(n) => Ok(n as f32), + None => Ok(t.raw_get::>(key)?.unwrap_or(0.0) as f32), + } + }; + Ok([get(1, "x")?, get(2, "y")?, get(3, "z")?]) + } + Value::Vector(v) => Ok([v.x(), v.y(), v.z()]), + _ => Err(lua_error(format!("{what} must be a {{x, y, z}} list"))), + } +} + +fn color_of(options: &Table, default: Token) -> mlua::Result { + Ok(parse_color(options.raw_get::("color")?) + .map_err(|e| lua_error(e.to_string()))? + .unwrap_or(Color::token(default))) +} + +/// A ramp through `colors` (a list of tokens) at `t` in 0..1. +fn ramp(colors: &[Color], t: f32) -> Color { + match colors.len() { + 0 => Color::token(Token::Accent), + 1 => colors[0], + n => { + let scaled = t.clamp(0.0, 1.0) * (n - 1) as f32; + let i = (scaled.floor() as usize).min(n - 2); + Color { + token: colors[i].token, + mix: Some((colors[i + 1].token, scaled - i as f32)), + alpha: colors[i].alpha, + } + } + } +} + +fn colors_of(options: &Table) -> mlua::Result> { + let Some(list) = options.raw_get::>("colors")? else { + return Ok(vec![color_of(options, Token::Accent)?]); + }; + let mut out = Vec::new(); + for value in list.sequence_values::() { + if let Some(color) = parse_color(value?).map_err(|e| lua_error(e.to_string()))? { + out.push(color); + } + } + Ok(out) +} + +impl UserData for Scene { + fn add_fields>(fields: &mut F) { + fields.add_field_method_get("time", |_, s| Ok(s.time)); + } + + fn add_methods>(methods: &mut M) { + methods.add_method_mut("box", |_, s, o: Table| { + let at = v3(o.raw_get("at")?, "box at")?; + let size = match o.raw_get::("size")? { + Value::Nil => [1.0; 3], + Value::Integer(n) => [n as f32; 3], + Value::Number(n) => [n as f32; 3], + other => v3(other, "box size")?, + }; + let color = color_of(&o, Token::Accent)?; + let h = scale(size, 0.5); + let c = |sx: f32, sy: f32, sz: f32| { + [at[0] + sx * h[0], at[1] + sy * h[1], at[2] + sz * h[2]] + }; + let quads = [ + ( + [ + c(-1., -1., 1.), + c(1., -1., 1.), + c(1., 1., 1.), + c(-1., 1., 1.), + ], + [0., 0., 1.], + ), + ( + [ + c(1., -1., -1.), + c(-1., -1., -1.), + c(-1., 1., -1.), + c(1., 1., -1.), + ], + [0., 0., -1.], + ), + ( + [ + c(1., -1., 1.), + c(1., -1., -1.), + c(1., 1., -1.), + c(1., 1., 1.), + ], + [1., 0., 0.], + ), + ( + [ + c(-1., -1., -1.), + c(-1., -1., 1.), + c(-1., 1., 1.), + c(-1., 1., -1.), + ], + [-1., 0., 0.], + ), + ( + [ + c(-1., 1., 1.), + c(1., 1., 1.), + c(1., 1., -1.), + c(-1., 1., -1.), + ], + [0., 1., 0.], + ), + ( + [ + c(-1., -1., -1.), + c(1., -1., -1.), + c(1., -1., 1.), + c(-1., -1., 1.), + ], + [0., -1., 0.], + ), + ]; + for (quad, outward) in quads { + s.face(&quad, color, Some(outward), false)?; + } + Ok(()) + }); + methods.add_method_mut("sphere", |_, s, o: Table| { + let at = v3(o.raw_get("at")?, "sphere at")?; + let r = o.raw_get::>("radius")?.unwrap_or(0.5).abs(); + let color = color_of(&o, Token::Accent)?; + let detail = o + .raw_get::>("detail")? + .unwrap_or(18) + .clamp(6, 48); + let (lon, lat) = (detail * 2, detail); + let p = |i: u32, j: u32| { + let theta = std::f32::consts::TAU * i as f32 / lon as f32; + let phi = std::f32::consts::PI * j as f32 / lat as f32; + [ + at[0] + r * phi.sin() * theta.cos(), + at[1] + r * phi.cos(), + at[2] + r * phi.sin() * theta.sin(), + ] + }; + for j in 0..lat { + for i in 0..lon { + let quad = [p(i, j), p(i + 1, j), p(i + 1, j + 1), p(i, j + 1)]; + let mid = scale(quad.iter().fold([0.0; 3], |a, v| add(a, *v)), 0.25); + let corners: Vec = if j == 0 { + vec![quad[0], quad[2], quad[3]] + } else if j == lat - 1 { + vec![quad[0], quad[1], quad[2]] + } else { + quad.to_vec() + }; + s.face(&corners, color, Some(sub(mid, at)), false)?; + } + } + Ok(()) + }); + methods.add_method_mut("cylinder", |_, s, o: Table| { + let at = v3(o.raw_get("at")?, "cylinder at")?; + let r = o.raw_get::>("radius")?.unwrap_or(0.5).abs(); + let h = o.raw_get::>("height")?.unwrap_or(1.0); + let color = color_of(&o, Token::Accent)?; + let n = o + .raw_get::>("detail")? + .unwrap_or(24) + .clamp(6, 64); + let ring = |i: u32, y: f32| { + let a = std::f32::consts::TAU * i as f32 / n as f32; + [at[0] + r * a.cos(), at[1] + y, at[2] + r * a.sin()] + }; + for i in 0..n { + let quad = [ring(i, 0.0), ring(i + 1, 0.0), ring(i + 1, h), ring(i, h)]; + let mid = scale(add(quad[0], quad[2]), 0.5); + s.face(&quad, color, Some(sub(mid, [at[0], mid[1], at[2]])), false)?; + } + let top: Vec = (0..n).map(|i| ring(i, h)).collect(); + let bottom: Vec = (0..n).map(|i| ring(i, 0.0)).collect(); + s.polygon(&top, color, Some([0.0, h.signum(), 0.0]), false)?; + s.polygon(&bottom, color, Some([0.0, -h.signum(), 0.0]), false) + }); + // A height field: y = f(x, z) over a grid, or `values` rows. + methods.add_method_mut("surface", |_, s, o: Table| { + let range = |key: &str| -> mlua::Result<(f32, f32)> { + match o.raw_get::>(key)? { + Some(t) => Ok((t.raw_get::(1)?, t.raw_get::(2)?)), + None => Ok((-1.0, 1.0)), + } + }; + let (x0, x1) = range("x")?; + let (z0, z1) = range("z")?; + let colors = colors_of(&o)?; + let wire = o.raw_get::>("wire")?.unwrap_or(false); + let mut grid: Vec> = Vec::new(); + if let Some(values) = o.raw_get::>("values")? { + for row in values.sequence_values::
() { + grid.push(row?.sequence_values::().collect::>()?); + } + } else { + let f: Function = o + .raw_get::>("f")? + .ok_or_else(|| lua_error("scene:surface needs f = function(x, z) or values"))?; + let n = o + .raw_get::>("detail")? + .unwrap_or(32) + .clamp(2, 120) as usize; + for j in 0..=n { + let z = z0 + (z1 - z0) * j as f32 / n as f32; + let mut row = Vec::with_capacity(n + 1); + for i in 0..=n { + let x = x0 + (x1 - x0) * i as f32 / n as f32; + row.push(f.call::((x, z))?); + } + grid.push(row); + } + } + let rows = grid.len(); + let cols = grid.iter().map(Vec::len).min().unwrap_or(0); + if rows < 2 || cols < 2 { + return Err(lua_error("scene:surface needs at least a 2x2 grid")); + } + let (lo, hi) = grid + .iter() + .flatten() + .fold((f32::INFINITY, f32::NEG_INFINITY), |(lo, hi), v| { + (lo.min(*v), hi.max(*v)) + }); + let span = (hi - lo).max(1e-6); + let point = |i: usize, j: usize| { + [ + x0 + (x1 - x0) * i as f32 / (cols - 1) as f32, + grid[j][i], + z0 + (z1 - z0) * j as f32 / (rows - 1) as f32, + ] + }; + for j in 0..rows - 1 { + for i in 0..cols - 1 { + let quad = [ + point(i, j), + point(i + 1, j), + point(i + 1, j + 1), + point(i, j + 1), + ]; + let mean = quad.iter().map(|v| v[1]).sum::() / 4.0; + let color = ramp(&colors, (mean - lo) / span); + if wire { + s.check_items()?; + for (a, b) in [(quad[0], quad[1]), (quad[0], quad[3])] { + s.lines.push(Line3 { + a, + b, + color, + width: 1.0, + }); + } + } else { + s.face(&quad, color, Some([0.0, 1.0, 0.0]), true)?; + } + } + } + Ok(()) + }); + methods.add_method_mut("mesh", |_, s, o: Table| { + let color = color_of(&o, Token::Accent)?; + let vertices: Vec = o + .raw_get::
("vertices")? + .sequence_values::() + .map(|v| v.and_then(|v| v3(v, "mesh vertex"))) + .collect::>()?; + for face in o.raw_get::
("faces")?.sequence_values::
() { + let corners: Vec = face? + .sequence_values::() + .map(|i| { + i.and_then(|i| { + vertices.get(i.wrapping_sub(1)).copied().ok_or_else(|| { + lua_error(format!("mesh face index {i} is out of range")) + }) + }) + }) + .collect::>()?; + s.polygon(&corners, color, None, true)?; + } + Ok(()) + }); + methods.add_method_mut("line", |_, s, (a, b, o): (Value, Value, Option
)| { + s.check_items()?; + let (a, b, o) = match (&a, &b) { + (Value::Table(t), Value::Nil) + if t.raw_get::("from").is_ok_and(|v| !v.is_nil()) => + { + (t.raw_get("from")?, t.raw_get("to")?, Some(t.clone())) + } + _ => (a, b, o), + }; + let (color, width) = line_style(o)?; + s.lines.push(Line3 { + a: v3(a, "line start")?, + b: v3(b, "line end")?, + color, + width, + }); + Ok(()) + }); + methods.add_method_mut("path", |_, s, (points, o): (Table, Option
)| { + let (color, width) = line_style(o)?; + let pts: Vec = points + .sequence_values::() + .map(|v| v.and_then(|v| v3(v, "path point"))) + .collect::>()?; + for pair in pts.windows(2) { + s.check_items()?; + s.lines.push(Line3 { + a: pair[0], + b: pair[1], + color, + width, + }); + } + Ok(()) + }); + methods.add_method_mut("point", |_, s, (at, o): (Value, Option
)| { + s.check_items()?; + let (at, o) = options_form(at, o)?; + let (color, radius, hint) = match o { + Some(o) => ( + color_of(&o, Token::Accent)?, + o.raw_get::>("radius")? + .unwrap_or(3.0) + .clamp(0.5, 24.0), + o.raw_get::("hint") + .map(|v| (!v.is_nil()).then(|| value_text(&v)))?, + ), + None => (Color::token(Token::Accent), 3.0, None), + }; + s.points.push(Point3 { + at: v3(at, "point")?, + radius, + color, + hint, + }); + Ok(()) + }); + methods.add_method_mut( + "label", + |_, s, (at, text, o): (Value, Value, Option
)| { + s.check_items()?; + let (at, text, o) = match (&at, &text) { + (Value::Table(t), Value::Nil) + if t.raw_get::("at").is_ok_and(|v| !v.is_nil()) => + { + (t.raw_get("at")?, t.raw_get("text")?, Some(t.clone())) + } + _ => (at, text, o), + }; + let (color, size) = match o { + Some(o) => ( + color_of(&o, Token::Muted)?, + match o.raw_get::>("size")? { + Some(size) => { + TextSize::parse(&size).map_err(|e| lua_error(e.to_string()))? + } + None => TextSize::Xs, + }, + ), + None => (Color::token(Token::Muted), TextSize::Xs), + }; + s.labels.push(Label3 { + at: v3(at, "label")?, + text: value_text(&text), + color, + size, + }); + Ok(()) + }, + ); + // A floor grid on y = 0 in hairlines. + methods.add_method_mut("grid", |_, s, o: Option
| { + let (size, step, y) = match &o { + Some(o) => ( + o.raw_get::>("size")? + .unwrap_or(10.0) + .abs() + .max(0.1), + o.raw_get::>("step")? + .unwrap_or(1.0) + .abs() + .max(0.01), + o.raw_get::>("y")?.unwrap_or(0.0), + ), + None => (10.0, 1.0, 0.0), + }; + let half = size / 2.0; + let n = ((size / step).round() as i32).clamp(1, 200); + for i in 0..=n { + s.check_items()?; + let t = -half + size * i as f32 / n as f32; + let color = Color::token(Token::Border); + s.lines.push(Line3 { + a: [t, y, -half], + b: [t, y, half], + color, + width: 1.0, + }); + s.lines.push(Line3 { + a: [-half, y, t], + b: [half, y, t], + color, + width: 1.0, + }); + } + Ok(()) + }); + // x/y/z axes from the origin, with optional names. + methods.add_method_mut("axes", |_, s, o: Option
| { + let size = match &o { + Some(o) => o.raw_get::>("size")?.unwrap_or(1.0), + None => 1.0, + }; + let names: [String; 3] = match &o { + Some(o) => match o.raw_get::>("labels")? { + Some(t) => [ + t.raw_get::>(1)?.unwrap_or_default(), + t.raw_get::>(2)?.unwrap_or_default(), + t.raw_get::>(3)?.unwrap_or_default(), + ], + None => ["x".into(), "y".into(), "z".into()], + }, + None => ["x".into(), "y".into(), "z".into()], + }; + for (axis, name) in names.into_iter().enumerate() { + s.check_items()?; + let mut end = [0.0; 3]; + end[axis] = size; + s.lines.push(Line3 { + a: [0.0; 3], + b: end, + color: Color::token(Token::Muted), + width: 1.0, + }); + if !name.is_empty() { + end[axis] = size * 1.08; + s.labels.push(Label3 { + at: end, + text: name, + color: Color::token(Token::Muted), + size: TextSize::Xs, + }); + } + } + Ok(()) + }); + } +} + +/// `s:point(at, options)` or `s:point { at = ..., ... }`. +fn options_form(at: Value, o: Option
) -> mlua::Result<(Value, Option
)> { + if let Value::Table(t) = &at + && o.is_none() + && let at_field @ Value::Table(_) = t.raw_get::("at")? + { + return Ok((at_field, Some(t.clone()))); + } + Ok((at, o)) +} + +fn line_style(o: Option
) -> mlua::Result<(Color, f32)> { + match o { + Some(o) => Ok(( + color_of(&o, Token::Muted)?, + o.raw_get::>("width")? + .unwrap_or(1.0) + .clamp(0.25, 12.0), + )), + None => Ok((Color::token(Token::Muted), 1.0)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cube() -> Scene { + let mut s = Scene::default(); + let h = 0.5; + let c = |x: f32, y: f32, z: f32| [x * h, y * h, z * h]; + s.face( + &[ + c(-1., -1., 1.), + c(1., -1., 1.), + c(1., 1., 1.), + c(-1., 1., 1.), + ], + Color::token(Token::Accent), + Some([0., 0., 1.]), + false, + ) + .unwrap(); + s.face( + &[ + c(1., -1., -1.), + c(-1., -1., -1.), + c(-1., 1., -1.), + c(1., 1., -1.), + ], + Color::token(Token::Accent), + Some([0., 0., -1.]), + false, + ) + .unwrap(); + s + } + + #[test] + fn the_camera_frames_the_geometry_and_culls_back_faces() { + let projection = cube().project( + Camera { + yaw: 0.0, + pitch: 0.0, + zoom: 1.0, + }, + 400.0, + 300.0, + ); + // Looking down -z from +z: only the +z face is drawn. + assert_eq!(projection.prims.len(), 1); + let Prim::Face { points, shade, .. } = &projection.prims[0] else { + panic!() + }; + for [x, y] in points { + assert!( + (0.0..=400.0).contains(x) && (0.0..=300.0).contains(y), + "{x},{y}" + ); + } + assert!(*shade > 0.35); + // From behind, the other face. + let behind = cube().project( + Camera { + yaw: std::f32::consts::PI, + pitch: 0.0, + zoom: 1.0, + }, + 400.0, + 300.0, + ); + assert_eq!(behind.prims.len(), 1); + } + + #[test] + fn faces_sort_back_to_front() { + let mut s = Scene::default(); + let near = [[0., 0., 1.], [1., 0., 1.], [1., 1., 1.]]; + let far = [[0., 0., -1.], [1., 0., -1.], [1., 1., -1.]]; + s.face(&near, Color::token(Token::Chart(0)), None, true) + .unwrap(); + s.face(&far, Color::token(Token::Chart(1)), None, true) + .unwrap(); + let p = s.project( + Camera { + yaw: 0.0, + pitch: 0.0, + zoom: 1.0, + }, + 200.0, + 200.0, + ); + let colors: Vec = p + .prims + .iter() + .map(|p| match p { + Prim::Face { color, .. } => color.token, + _ => unreachable!(), + }) + .collect(); + assert_eq!(colors, [Token::Chart(1), Token::Chart(0)]); + } + + #[test] + fn ramps_mix_between_neighbouring_tokens() { + let colors = [ + Color::token(Token::Chart(0)), + Color::token(Token::Chart(1)), + Color::token(Token::Chart(2)), + ]; + let mid = ramp(&colors, 0.75); + assert_eq!(mid.token, Token::Chart(1)); + assert_eq!(mid.mix.unwrap().0, Token::Chart(2)); + assert!((mid.mix.unwrap().1 - 0.5).abs() < 1e-5); + } +} diff --git a/crates/visual/src/style.rs b/crates/visual/src/style.rs new file mode 100644 index 0000000000..c0cc82f02a --- /dev/null +++ b/crates/visual/src/style.rs @@ -0,0 +1,267 @@ +//! The style vocabulary a visual may use. Every value names something in the +//! active Zeron theme (a color token, a step on the transcript's type scale), +//! so a script can't paint a background, pick a font, or hard-code a color: +//! the renderer resolves tokens against whatever theme, surface (glass or +//! opaque), appearance, and text size the reader has. + +use crate::VisualError; + +/// A theme color a visual may use. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Token { + /// Primary text. + Text, + /// Labels, axes, captions. + Muted, + /// Tertiary notes. + Faint, + /// The one thing to notice; also the first chart series. + Accent, + /// Text drawn on an accent mark. + OnAccent, + Success, + Warning, + Danger, + /// Hairlines: rules, gridlines, axes. + Border, + /// Categorical series colors, `chart1` … `chart6` (0-based here). + Chart(u8), +} + +pub const CHART_SERIES: u8 = 6; + +/// Every token name a script may write, for error messages and the agent +/// guidance. +pub const TOKEN_NAMES: [&str; 15] = [ + "text", + "muted", + "faint", + "accent", + "on-accent", + "success", + "warning", + "danger", + "border", + "chart1", + "chart2", + "chart3", + "chart4", + "chart5", + "chart6", +]; + +impl Token { + pub fn parse(name: &str) -> Result { + Ok(match name { + "text" => Token::Text, + "muted" => Token::Muted, + "faint" => Token::Faint, + "accent" => Token::Accent, + "on-accent" => Token::OnAccent, + "success" => Token::Success, + "warning" => Token::Warning, + "danger" => Token::Danger, + "border" => Token::Border, + other => match other + .strip_prefix("chart") + .and_then(|n| n.parse::().ok()) + { + Some(n @ 1..=CHART_SERIES) => Token::Chart(n - 1), + _ => { + return Err(VisualError::script(format!( + "unknown color '{other}': colors are theme tokens ({}), so a visual \ + reads on glass and opaque, light and dark", + TOKEN_NAMES.join(", ") + ))); + } + }, + }) + } + + /// The `index`th categorical series color (0-based, cycling). + pub fn series(index: usize) -> Self { + Token::Chart((index % CHART_SERIES as usize) as u8) + } +} + +/// A resolved-at-paint color: a token, optionally mixed toward a second +/// token, at an opacity. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Color { + pub token: Token, + pub mix: Option<(Token, f32)>, + pub alpha: f32, +} + +impl Color { + pub const fn token(token: Token) -> Self { + Self { + token, + mix: None, + alpha: 1.0, + } + } + + pub fn with_alpha(self, alpha: f32) -> Self { + Self { + alpha: self.alpha * alpha.clamp(0.0, 1.0), + ..self + } + } +} + +/// A step on the transcript's type scale. Sizes are design pixels at the +/// 16px interface base; the renderer scales them with the reader's text size. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TextSize { + Xs, + Sm, + #[default] + Body, + Lg, + Xl, + Xxl, +} + +impl TextSize { + pub fn parse(name: &str) -> Result { + Ok(match name { + "xs" => TextSize::Xs, + "sm" => TextSize::Sm, + "body" | "md" | "base" => TextSize::Body, + "lg" => TextSize::Lg, + "xl" => TextSize::Xl, + "2xl" => TextSize::Xxl, + other => { + return Err(VisualError::script(format!( + "unknown text size '{other}': use xs, sm, body, lg, xl, or 2xl (the \ + transcript's type scale)" + ))); + } + }) + } + + /// (font size, line height) in design pixels at the 16px base, the same + /// steps the transcript's markdown uses. + pub fn metrics(self) -> (f32, f32) { + match self { + TextSize::Xs => (11.0, 16.0), + TextSize::Sm => (12.0, 18.0), + TextSize::Body => (14.0, 22.0), + TextSize::Lg => (15.0, 22.0), + TextSize::Xl => (16.0, 24.0), + TextSize::Xxl => (19.0, 27.0), + } + } +} + +/// Weights the interface font carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Weight { + #[default] + Regular, + Medium, + Semibold, + Bold, +} + +impl Weight { + pub fn from_number(weight: f64) -> Self { + match weight as i64 { + ..=449 => Weight::Regular, + 450..=549 => Weight::Medium, + 550..=649 => Weight::Semibold, + _ => Weight::Bold, + } + } + + pub fn parse(name: &str) -> Result { + Ok(match name { + "regular" | "normal" => Weight::Regular, + "medium" => Weight::Medium, + "semibold" => Weight::Semibold, + "bold" => Weight::Bold, + other => { + return Err(VisualError::script(format!( + "unknown weight '{other}': use 400-700 or regular, medium, semibold, bold" + ))); + } + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Align { + #[default] + Start, + Center, + End, +} + +impl Align { + pub fn parse(name: &str) -> Result { + Ok(match name { + "start" | "left" | "top" => Align::Start, + "center" | "middle" => Align::Center, + "end" | "right" | "bottom" => Align::End, + other => { + return Err(VisualError::script(format!( + "unknown alignment '{other}': use start, center, or end" + ))); + } + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Justify { + #[default] + Start, + Center, + End, + Between, +} + +impl Justify { + pub fn parse(name: &str) -> Result { + Ok(match name { + "between" | "space-between" => Justify::Between, + other => match Align::parse(other)? { + Align::Start => Justify::Start, + Align::Center => Justify::Center, + Align::End => Justify::End, + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn colors_are_theme_tokens_only() { + assert_eq!(Token::parse("chart1").unwrap(), Token::Chart(0)); + assert_eq!(Token::parse("chart6").unwrap(), Token::Chart(5)); + for bad in [ + "chart0", + "chart7", + "#ff0000", + "red", + "rgb(0,0,0)", + "background", + ] { + let err = Token::parse(bad).unwrap_err().to_string(); + assert!(err.contains("theme tokens"), "{bad}: {err}"); + } + assert_eq!(Token::series(7), Token::Chart(1)); + } + + #[test] + fn sizes_follow_the_transcript_scale() { + assert_eq!(TextSize::parse("body").unwrap().metrics(), (14.0, 22.0)); + assert_eq!(TextSize::parse("2xl").unwrap().metrics(), (19.0, 27.0)); + assert!(TextSize::parse("40px").is_err()); + assert_eq!(Weight::from_number(600.0), Weight::Semibold); + } +} diff --git a/crates/visual/src/tests.rs b/crates/visual/src/tests.rs new file mode 100644 index 0000000000..4f0dde7c79 --- /dev/null +++ b/crates/visual/src/tests.rs @@ -0,0 +1,278 @@ +use super::*; + +fn text_of(node: &Node) -> String { + match &node.kind { + NodeKind::Text(text) => text.spans.iter().map(|s| s.text.as_str()).collect(), + _ => children(node).map(text_of).collect::>().join("|"), + } +} + +fn find<'a>(node: &'a Node, pred: &dyn Fn(&Node) -> bool) -> Option<&'a Node> { + if pred(node) { + return Some(node); + } + children(node).find_map(|child| find(child, pred)) +} + +#[test] +fn a_view_renders_and_handlers_change_state() { + let mut visual = Visual::load( + r#" + local s = ui.state { tab = "p95" } + return function() + return ui.column { gap = 12, + ui.text { "Latency ", ui.span { s.tab, weight = 600, color = "accent" }, size = "lg" }, + ui.tabs { options = { "p95", "p99" }, value = s.tab, on_change = function(v) s.tab = v end }, + } + end + "#, + 1, + ) + .unwrap(); + let tree = visual.render().unwrap(); + assert_eq!(text_of(&tree), "Latency p95|"); + let tabs = find(&tree, &|n| matches!(n.kind, NodeKind::Tabs { .. })).unwrap(); + let NodeKind::Tabs { on_change, .. } = tabs.kind else { + unreachable!() + }; + visual + .call(on_change, HandlerArg::Text("p99".into())) + .unwrap(); + assert_eq!(text_of(&visual.render().unwrap()), "Latency p99|"); +} + +#[test] +fn a_script_may_return_a_static_node() { + let mut visual = Visual::load(r#"return ui.text "hello""#, 1).unwrap(); + assert_eq!(text_of(&visual.render().unwrap()), "hello"); +} + +#[test] +fn style_rules_are_enforced_with_reasons() { + for (script, needle) in [ + ( + r##"return ui.text { "x", color = "#ff0000" }"##, + "theme tokens", + ), + ( + r#"return ui.column { background = "accent" }"#, + "no containers", + ), + ( + r#"return ui.text { "x", font = "Comic Sans" }"#, + "interface font", + ), + (r#"return ui.text { "x", size = "40px" }"#, "type scale"), + (r#"return ui.row { colour = "x" }"#, "has no 'colour'"), + ( + r#"return ui.chart { type = "pie", series = {} }"#, + "ui.canvas", + ), + ( + r#"return ui.canvas { draw = function() end }"#, + "needs height", + ), + ] { + let err = Visual::load(script, 1) + .and_then(|mut v| v.render()) + .unwrap_err(); + assert!(err.message.contains(needle), "{script}: {}", err.message); + } +} + +#[test] +fn the_sandbox_has_no_io_and_bounds_time_and_memory() { + for script in [ + "return os.execute('true')", + "return io.open('/etc/passwd')", + "return require('x')", + ] { + assert!(Visual::load(script, 1).is_err(), "{script}"); + } + let started = Instant::now(); + let err = Visual::load("while true do end", 1).err().unwrap(); + assert!(err.message.contains("time budget"), "{}", err.message); + assert!(started.elapsed() < Duration::from_secs(2)); + + let err = Visual::load( + "local t = {} for i = 1, 1e9 do t[i] = string.rep('x', 1024) .. i end", + 1, + ) + .err() + .unwrap(); + assert!( + err.message.contains("MB") || err.message.contains("time budget"), + "{}", + err.message + ); + // The ui library can't be replaced by a script. + assert!(Visual::load("ui.text = nil return ui.text 'x'", 1).is_err()); +} + +#[test] +fn random_data_is_stable_across_renders() { + let script = r#" + return function() return ui.text { tostring(math.random(1, 1000000)) } end + "#; + let mut a = Visual::load(script, 7).unwrap(); + let first = text_of(&a.render().unwrap()); + assert_eq!(first, text_of(&a.render().unwrap())); + let mut b = Visual::load(script, 7).unwrap(); + assert_eq!(first, text_of(&b.render().unwrap())); +} + +#[test] +fn canvases_record_theme_colored_commands() { + let mut visual = Visual::load( + r#" + return ui.canvas { height = 120, draw = function(g, w, h) + g:line(0, h, w, 0, { color = "border" }) + g:rect(10, 10, 40, 20, { fill = "chart2", radius = 4 }) + g:circle(w / 2, h / 2, 8, { color = ui.color("accent", 0.5) }) + g:arc(60, 60, 30, 0, math.pi, { fill = "chart3", inner = 20 }) + g:text(w, 0, "label", { align = "end", color = "muted", size = "xs" }) + end } + "#, + 1, + ) + .unwrap(); + let tree = visual.render().unwrap(); + let NodeKind::Canvas(canvas) = &tree.kind else { + panic!() + }; + assert_eq!(canvas.height, 120.0); + let commands = visual + .draw_canvas(canvas.draw, 300.0, 120.0, 0.0, None) + .unwrap(); + assert_eq!(commands.len(), 5); + assert!(matches!(&commands[0], DrawCmd::Stroke { color, .. } if color.token == Token::Border)); + assert!(matches!(&commands[2], DrawCmd::Fill { color, .. } if color.alpha == 0.5)); + let DrawCmd::Text(text) = &commands[4] else { + panic!() + }; + assert_eq!((text.x, text.align), (300.0, Align::End)); +} + +#[test] +fn scenes_project_lit_depth_sorted_geometry() { + let mut visual = Visual::load( + r#" + return ui.scene { height = 300, spin = 0.3, draw = function(s) + s:grid { size = 4, step = 1 } + s:surface { x = { -2, 2 }, z = { -2, 2 }, detail = 12, + colors = { "chart1", "chart3" }, + f = function(x, z) return math.sin(x) * math.cos(z) end } + s:box { at = { 0, 1.5, 0 }, size = 0.5, color = "chart2" } + s:point({ 1, 1, 1 }, { hint = "peak" }) + s:point { at = { -1, 1, -1 }, color = "chart4" } + s:label { at = { 0, 2, 0 }, text = "top" } + end } + "#, + 1, + ) + .unwrap(); + let tree = visual.render().unwrap(); + let NodeKind::Scene(node) = &tree.kind else { + panic!() + }; + let scene = visual.draw_scene(node.draw, 0.0).unwrap(); + assert_eq!(scene.face_count(), 12 * 12 + 6); + let projection = scene.project( + Camera { + yaw: node.yaw, + pitch: node.pitch, + zoom: node.zoom, + }, + 640.0, + 300.0, + ); + assert!(matches!(projection.prims.last(), Some(Prim::Label { .. }))); + assert_eq!(projection.hints.len(), 1); + // A closed box shows at most three faces. + let box_faces = projection + .prims + .iter() + .filter(|p| matches!(p, Prim::Face { color, .. } if color.token == Token::Chart(1))) + .count(); + assert!((1..=3).contains(&box_faces), "{box_faces}"); +} + +#[test] +fn charts_and_tables_parse() { + let mut visual = Visual::load( + r#" + return ui.column { + ui.chart { type = "line", height = 200, x = { "Mon", "Tue", "Wed" }, unit = "ms", + series = { { name = "p95", values = { 120, 180, 140 } }, { name = "p99", values = { 200, nil, 260 } } } }, + ui.table { columns = { "Region", "p95" }, rows = { { "us-east", 182 }, { "eu-west", 140 } } }, + ui.row { ui.stat { value = "182 ms", label = "p95" }, ui.stat { value = 99.2, label = "uptime", tone = "success" } }, + } + "#, + 1, + ) + .unwrap(); + let tree = visual.render().unwrap(); + let chart = find(&tree, &|n| matches!(n.kind, NodeKind::Chart(_))).unwrap(); + let NodeKind::Chart(chart) = &chart.kind else { + unreachable!() + }; + assert_eq!(chart.series[1].values, [Some(200.0), None, Some(260.0)]); + assert_eq!(chart.series[1].color, Color::token(Token::Chart(1))); + let table = find(&tree, &|n| matches!(n.kind, NodeKind::Table(_))).unwrap(); + let NodeKind::Table(table) = &table.kind else { + unreachable!() + }; + // Numeric columns right-align. + assert_eq!(table.columns[1].align, Align::End); +} + +#[test] +fn check_exercises_every_control_and_reports_where_it_broke() { + let report = check( + r#" + local s = ui.state { tab = "a", on = false, n = 1 } + return function() + return ui.column { + ui.tabs { options = { "a", "b" }, value = s.tab, on_change = function(v) s.tab = v end }, + ui.toggle { label = "Log scale", value = s.on, on_change = function(v) s.on = v end }, + ui.slider { label = "Window", value = s.n, min = 1, max = 10, on_change = function(v) s.n = v end }, + ui.canvas { height = 100, animate = true, draw = function(g, w, h, t) g:line(0, 0, w * t, h) end }, + } + end + "#, + 1, + ) + .unwrap(); + assert_eq!(report.interactions, 5); + + let err = check( + r#" + local s = ui.state { tab = "a" } + return function() + local body = s.tab == "a" and ui.text "fine" or ui.text { nil .. "boom" } + return ui.column { + ui.tabs { options = { "a", "b" }, value = s.tab, on_change = function(v) s.tab = v end }, + body, + } + end + "#, + 1, + ) + .unwrap_err(); + assert!( + err.message.starts_with("after selecting tab 'b':"), + "{}", + err.message + ); + assert!(err.message.contains("visual:4"), "{}", err.message); +} + +#[test] +fn ticks_are_round() { + assert_eq!( + nice_ticks(0.0, 100.0, 5), + [0.0, 20.0, 40.0, 60.0, 80.0, 100.0] + ); + assert_eq!(nice_ticks(0.0, 0.9, 4), [0.0, 0.2, 0.4, 0.6, 0.8]); + assert_eq!(nice_ticks(5.0, 5.0, 4), [5.0]); +} diff --git a/docs/mcp.md b/docs/mcp.md index 0cf820a675..3086517949 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -125,6 +125,7 @@ name (default: the local engine's device). | `interrupt_chat` | `QueueCommand` Interrupt | | `respond_to_input` | `QueueCommand` RespondInput | | `archive_chat` | `Mutate setChatArchived` | +| `show_visual` | `PublishVisual {chatId, title, script}` (origin chat only) | Watch streams are the engine's only read surface (there is no one-shot "get transcript" RPC); a snapshot is "subscribe, take the first item, drop" — drop @@ -166,6 +167,50 @@ edge. A brand-new chat has no session row until the host picks the run up, so the wait keeps waiting in that case rather than reporting the unstarted run as done (this was the one bug the first live run found). +### Visual replies + +`show_visual` is listed only when `ZERON_CHAT_ID` is set: it shows an +interactive visual (chart, table, dashboard, diagram, simulation, 3D view) +inline in the calling chat, above the agent's final reply. A visual is a Luau +script, not a web page. The script builds its data and state once and returns a +view function; the view returns a tree of `ui.*` nodes (rows, columns, grids, +text, stats, tables, charts, tabs, toggles, sliders, buttons, links, 2D +canvases, and 3D scenes). Controls call the script's handlers, and Zeron renders +the view again. + +`crates/visual` (`zeron-visual`) is the runtime. Scripts run in a Luau sandbox +with only the `string`, `table`, `math`, `utf8` and `bit32` libraries (no I/O, +network, or `require`), a 64 MB memory cap, and a time budget per call (250 ms +for setup, renders and handlers, 60 ms per canvas or scene frame), so a runaway +script fails with an error instead of freezing the app. `math.random` is +reseeded from the visual's id before every call, so random data is identical on +every render and device. The style rules are the API rather than advice: colors +are theme tokens (`text`, `muted`, `faint`, `accent`, `on-accent`, `success`, +`warning`, `danger`, `border`, `chart1`…`chart6`), text sizes are steps on the +transcript's type scale, the font is always the interface font, and layout nodes +have no fill, border, shadow, padding, or font props; using one is an error that +names the rule. 3D scenes are projected in the runtime: Zeron frames an orbit +camera on the geometry, lights faces with one soft light, culls and depth-sorts +them, and hands the desktop 2D polygons. + +The engine runs every script before showing it (`zeron_visual::check`: setup, +the first render, every canvas and scene draw, then each tab, toggle, slider +end, and button once) and returns the first error with its line, so the agent +fixes it before the reader sees anything. A passing script is stored in uploads +(`{id}-visual.luau`) and a `visual` doc part is folded into the chat's live turn +through the run's engine event channel; with no live turn on this device the +call fails and nothing is stored. The doc carries only the path and title, so +clients read the script with `ReadAttachmentChunk` like any attachment; old +clients see an empty part. + +The desktop (`crates/ui/src/visuals.rs`) renders the tree with the transcript's +own GPUI elements: markdown's table treatment, the settings tab, switch, and +button idioms, and canvases and scenes painted as GPUI paths and shaped text in +theme colors. Nothing foreign is hosted, so a visual scrolls, clips, blurs under +the glass composer, and takes input like any other row, identically on Linux, +macOS, and Windows, and it follows theme, appearance, and text-size changes +without rerunning the script. + ## Selecting a host and project `list_projects {device?}` filters by device id or exact name; without arguments From 771b40240d1d2f3c51c6ffc6dc395028bb73e51a Mon Sep 17 00:00:00 2001 From: Wing <44917547+wingleeio@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:17:23 -0700 Subject: [PATCH 2/4] Render visual scenes and shaders on the GPU - Scenes rasterize with a depth buffer (no more painter's-algorithm occlusion errors), clip at the near plane, shade curved solids smoothly, and paint as one image; the software path renders in parallel row bands at full resolution. - With a GPU (wgpu, new crate zeron-visual-gpu), scenes render with PBR materials (roughness, metal, glow, glass with ior and dispersion), soft shadows, MSAA, and bloom; the software renderer is the fallback. - ui.shader runs Shadertoy-style GLSL on the GPU: iTime, iResolution, iMouse, iCamera (orbit), named inputs bound to state, iNoise, and up to four buffer passes read as iChannel0-3 with feedback. Composited by blend (light by default, so black is clear; alpha; opaque) with optional bloom. Each pass is validated with naga so errors reach the agent with their line. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 22 + Cargo.toml | 9 + crates/mcp/src/tools.rs | 27 +- crates/ui/Cargo.toml | 1 + crates/ui/src/visuals.rs | 563 ++++++--- crates/visual-gpu/Cargo.toml | 14 + crates/visual-gpu/examples/render_shader.rs | 71 ++ crates/visual-gpu/src/lib.rs | 174 +++ crates/visual-gpu/src/scene.rs | 1242 +++++++++++++++++++ crates/visual-gpu/src/shader.rs | 720 +++++++++++ crates/visual-gpu/tests/mandelbulb.glsl | 50 + crates/visual/Cargo.toml | 2 + crates/visual/src/lib.rs | 15 +- crates/visual/src/node.rs | 117 ++ crates/visual/src/prelude.luau | 1 + crates/visual/src/raster.rs | 772 ++++++++++++ crates/visual/src/scene.rs | 296 ++++- crates/visual/src/shader.rs | 269 ++++ crates/visual/src/tests.rs | 61 + docs/mcp.md | 24 +- 20 files changed, 4243 insertions(+), 207 deletions(-) create mode 100644 crates/visual-gpu/Cargo.toml create mode 100644 crates/visual-gpu/examples/render_shader.rs create mode 100644 crates/visual-gpu/src/lib.rs create mode 100644 crates/visual-gpu/src/scene.rs create mode 100644 crates/visual-gpu/src/shader.rs create mode 100644 crates/visual-gpu/tests/mandelbulb.glsl create mode 100644 crates/visual/src/raster.rs create mode 100644 crates/visual/src/shader.rs diff --git a/Cargo.lock b/Cargo.lock index 5aa756f94e..7c3a813440 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6032,6 +6032,7 @@ dependencies = [ "log", "num-traits", "once_cell", + "pp-rs", "rustc-hash 1.1.0", "spirv", "thiserror 2.0.20", @@ -7688,6 +7689,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pp-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" +dependencies = [ + "unicode-xid", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -13907,6 +13917,7 @@ dependencies = [ "zeron-update", "zeron-veil", "zeron-visual", + "zeron-visual-gpu", "zeron-voice", "zeron-voice-media", "zeron-voice-session", @@ -13937,9 +13948,20 @@ name = "zeron-visual" version = "0.2.112" dependencies = [ "mlua", + "naga", "thiserror 2.0.20", ] +[[package]] +name = "zeron-visual-gpu" +version = "0.2.112" +dependencies = [ + "pollster 0.4.0", + "tracing", + "wgpu", + "zeron-visual", +] + [[package]] name = "zeron-voice" version = "0.2.112" diff --git a/Cargo.toml b/Cargo.toml index 33127f3a5e..694ae794de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "crates/mcp", "crates/text", "crates/visual", + "crates/visual-gpu", "apps/zeron", ] @@ -56,6 +57,7 @@ zeron-theme = { path = "crates/theme" } zeron-mcp = { path = "crates/mcp" } zeron-text = { path = "crates/text" } zeron-visual = { path = "crates/visual" } +zeron-visual-gpu = { path = "crates/visual-gpu" } # CRDT loro = "1.13" @@ -172,6 +174,13 @@ opt-level = 0 [profile.dev.package."*"] opt-level = 2 +# Scenes rasterize in software every frame the camera moves. +[profile.dev.package.zeron-visual] +opt-level = 3 + +[profile.dev.package.zeron-visual-gpu] +opt-level = 3 + # Mobile core (scripts/ios/build-core.sh). Always optimized — layout and sync # run on the phone's hot paths, so even Debug app builds link an -O core. [profile.mobile] diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs index 7e7dc89bc0..c8cb3905d9 100644 --- a/crates/mcp/src/tools.rs +++ b/crates/mcp/src/tools.rs @@ -307,15 +307,36 @@ radians clockwise from 3 o'clock), g:text(x, y, str, { size, color, weight, alig { { x, y }, ... }. g.pointer is { x, y, down } while the pointer is over the canvas (draw hover \ effects with it); t is seconds, and animate = true redraws every frame while visible.\n\ - ui.scene { height = 320, draw = function(s, t) ... end, orbit = true, spin = 0.3 (turntable \ -radians per second), camera = { yaw = 0.6, pitch = 0.45, zoom = 1 }, animate = bool }: real 3D, y up. \ -Zeron frames the camera on your geometry, the reader drags to orbit, and faces are lit and \ -depth-sorted. s:box { at = { x, y, z }, size = { w, h, d } or n, color }, s:sphere { at, radius, color }, \ +radians per second), camera = { yaw = 0.6, pitch = 0.45, zoom = 1 }, animate = bool }: real 3D, y up, \ +rendered on the GPU with depth, soft shadows from a key light, and bloom. Zeron frames the camera on \ +your geometry and the reader drags to orbit. Solids (box, sphere, cylinder, surface, mesh) take \ +material props: roughness (0 mirror to 1 matte), metal (0-1), glow (emits its color and blooms; \ +use for lamps, beams, highlights), glass = true (refracts what's behind it; ior = 1.5, dispersion \ += 0-1 splits it into a rainbow, as a prism does). Build real forms from several primitives (legs, \ +tops, rims), not single boxes, and give materials so surfaces read as wood, metal, or glass. s:box { at = { x, y, z }, size = { w, h, d } or n, color }, s:sphere { at, radius, color }, \ s:cylinder { at (base center), radius, height, color }, s:surface { x = { min, max }, z = { min, max }, \ f = function(x, z) return y end or values = { { ... } }, detail = 32, colors = { \"chart1\", \"chart3\" } \ (a ramp by height), wire = bool }, s:mesh { vertices = { { x, y, z }, ... }, faces = { { 1, 2, 3 }, ... }, \ color }, s:line(a, b, { color, width }), s:path(points, { color, width }), s:point(at, { radius, \ color, hint = \"shown on hover\" }), s:label(at, text, { color, size }), s:grid { size = 10, step = 1 }, \ s:axes { size = 1, labels = { \"x\", \"y\", \"z\" } }.\n\ +- ui.shader { height = 360, code = [[ GLSL ]], buffers = { [[ GLSL ]], ... }, inputs = { speed = \ +s.speed }, blend = \"alpha|light|opaque\", bloom = 0-3, orbit = bool, spin, camera = { yaw, pitch, \ +zoom }, animate = true }: GPU shaders for what geometry can't express: raymarched SDFs and \ +fractals, light and optics (beams, refraction, dispersion), volumes, fluids, particles, generative \ +art. Write each pass as on Shadertoy: void mainImage(out vec4 fragColor, in vec2 fragCoord), with \ +iResolution (vec3, pixels), iTime, iTimeDelta, iFrame, iMouse (pixels from the bottom-left; xy while \ +pressed, zw where pressed), iCamera (vec4 yaw, pitch, zoom, 0; orbit = true lets the reader drag \ +it), every inputs name as a float (bind them to state so sliders and toggles drive the shader), \ +iNoise (a tiling 256x256 random RGBA texture), and iChannel0-iChannel3 with iChannelResolution: \ +buffers run first, in order, each into a float texture that every pass reads as iChannel (an \ +earlier buffer's current frame; a buffer's own previous frame), for multi-pass work like blur, \ +trails, accumulation, and simulation. The shader sits on the chat, never in a box: don't paint a \ +background, vignette, or floor color. By default (blend = \"light\") the output is emitted light: \ +brightness is opacity and black is clear, right for beams, glows, sparks, and light through glass. \ +For solid shaded objects (fractals, SDF bodies) set blend = \"alpha\" and write alpha 0 where the ray \ +misses. bloom = 0.6-1.5 adds a soft glow around bright light. \ +GLSL errors come back with the pass and line number.\n\ - Helpers: ui.scale(d0, d1, r0, r1) returns a mapping function; ui.ticks(lo, hi, n); ui.fmt(n, { unit, \ prefix, decimals }) groups digits; ui.map(list, function(v, i) end); ui.range(a, b, step); ui.clamp; \ ui.lerp; ui.series(i) is \"chart\"; ui.color(token, alpha); ui.mix(a, b, t). print(...) output comes \ diff --git a/crates/ui/Cargo.toml b/crates/ui/Cargo.toml index 44ba704ae4..c65c0dadcf 100644 --- a/crates/ui/Cargo.toml +++ b/crates/ui/Cargo.toml @@ -75,6 +75,7 @@ zeron-update.workspace = true zeron-syntax.workspace = true zeron-theme.workspace = true zeron-visual.workspace = true +zeron-visual-gpu.workspace = true gpui.workspace = true gpui_platform.workspace = true gpui_tokio.workspace = true diff --git a/crates/ui/src/visuals.rs b/crates/ui/src/visuals.rs index b7b28c666d..a61a4f2763 100644 --- a/crates/ui/src/visuals.rs +++ b/crates/ui/src/visuals.rs @@ -26,7 +26,7 @@ use gpui::{ TextAlign, TextRun, Window, canvas, div, point, px, }; use zeron_visual::{ - Align, ChartKind, DrawCmd, HandlerArg, HandlerId, Justify, Node, NodeKind, Prim, Token, + Align, ChartKind, DrawCmd, HandlerArg, HandlerId, Justify, Node, NodeKind, Token, }; use crate::markdown::render::LinkUi; @@ -72,9 +72,21 @@ struct Local { cameras: HashMap, canvases: HashMap, scenes: HashMap, + shaders: HashMap, draw_error: Option, } +/// A shader's last frame and what it was drawn with. +struct ShaderCache { + uniforms: zeron_visual::shader::ShaderUniforms, + size: (u32, u32), + image: Arc, + frame: u32, + last_time: f64, + /// Buffers carried between frames (feedback, simulations). + gpu: Option, +} + struct CanvasCache { generation: u64, size: (f32, f32), @@ -85,6 +97,30 @@ struct CanvasCache { struct SceneCache { generation: u64, scene: Rc, + /// The last image rendered from `scene`, and what it was rendered for. + image: Option<(RasterKey, SceneImage)>, + /// How long the last supersampled render took: cheap scenes stay + /// supersampled while the camera moves. + final_cost: Option, +} + +/// What a scene image depends on besides the geometry. +#[derive(Clone, Copy, PartialEq)] +struct RasterKey { + yaw: f32, + pitch: f32, + zoom: f32, + size: (f32, f32), + pixel_scale: f32, + quality: zeron_visual::Quality, + palette: u64, +} + +#[derive(Clone)] +struct SceneImage { + image: Arc, + hints: Rc>, + labels: Rc>, } #[derive(Clone, Copy)] @@ -354,14 +390,6 @@ fn mix_rgb(a: Hsla, b: Hsla, t: f32) -> Hsla { .into() } -/// A face's color under the scene light. -fn shaded(color: Hsla, shade: f32) -> Hsla { - Hsla { - l: (color.l * (0.42 + 0.62 * shade)).clamp(0.0, 0.97), - ..color - } -} - fn weight(weight: zeron_visual::Weight) -> FontWeight { match weight { zeron_visual::Weight::Regular => FontWeight::NORMAL, @@ -504,6 +532,7 @@ impl VisualFrame { NodeKind::Chart(chart) => self.chart(chart, key, theme, palette, cx), NodeKind::Canvas(canvas) => self.canvas(canvas, key, cx), NodeKind::Scene(scene) => self.scene(scene, key, cx), + NodeKind::Shader(shader) => self.shader(shader, key, cx), NodeKind::Button { label, on_click, @@ -1350,6 +1379,242 @@ fn paint_commands( // ---- 3D ------------------------------------------------------------------- impl VisualFrame { + /// Drag to orbit a scene's or shader's camera, picking up from where a + /// turntable `spin` has turned it. + fn orbit_handlers( + &self, + mut el: gpui::Stateful, + key: &SharedString, + spin: f32, + cx: &mut Context, + ) -> gpui::Stateful { + let (down_key, drag_key, move_key, up_key, out_key) = ( + key.clone(), + key.clone(), + key.clone(), + key.clone(), + key.clone(), + ); + let release = move |this: &mut VisualFrame, key: &SharedString| { + if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(key) + && orbit.drag.take().is_some() + { + orbit.spin_epoch = Instant::now(); + } + }; + el = el + .cursor_grab() + .on_mouse_down( + MouseButton::Left, + cx.listener(move |this, event: &gpui::MouseDownEvent, _, cx| { + cx.stop_propagation(); + if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(&down_key) { + orbit.yaw += spin * orbit.spin_epoch.elapsed().as_secs_f32(); + orbit.drag = Some((event.position, orbit.yaw, orbit.pitch)); + } + cx.notify(); + }), + ) + .on_drag(VisualDrag(drag_key), |_, _, _, cx| cx.new(|_| DragGhost)) + .on_drag_move(cx.listener( + move |this, event: &gpui::DragMoveEvent, _, cx| { + if event.drag(cx).0 != move_key { + return; + } + if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(&move_key) + && let Some((start, yaw, pitch)) = orbit.drag + { + let delta = event.event.position - start; + orbit.yaw = yaw - f32::from(delta.x) * 0.01; + orbit.pitch = (pitch + f32::from(delta.y) * 0.01).clamp(-1.5, 1.5); + } + cx.notify(); + }, + )) + .on_mouse_up( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + release(this, &up_key); + cx.notify(); + }), + ) + .on_mouse_up_out( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + release(this, &out_key); + cx.notify(); + }), + ); + el + } + + /// A `ui.shader`: the script's GLSL run on the GPU every frame it + /// animates, painted as an image like any other. + fn shader( + &mut self, + node: &zeron_visual::ShaderNode, + key: SharedString, + cx: &mut Context, + ) -> AnyElement { + let spin = if cx.reduce_motion() { 0.0 } else { node.spin }; + let animate = node.animate && !cx.reduce_motion(); + let orbit = *self + .local + .borrow_mut() + .cameras + .entry(key.clone()) + .or_insert(Orbit { + yaw: node.yaw, + pitch: node.pitch, + spin_epoch: Instant::now(), + drag: None, + }); + let yaw = match orbit.drag { + Some(_) => orbit.yaw, + None => orbit.yaw + spin * orbit.spin_epoch.elapsed().as_secs_f32(), + }; + let moving = animate || (spin != 0.0 && orbit.drag.is_none()); + let mut el = div() + .id(self.element_id(&format!("{key}-shader"))) + .w_full() + .h(px(node.height)); + el = self.pointer_tracking(el, &key, cx); + if node.orbit { + el = self.orbit_handlers(el, &key, spin, cx); + } + let names: Vec = node.inputs.iter().map(|(n, _)| n.clone()).collect(); + let mut inputs = [[0.0f32; 4]; zeron_visual::shader::MAX_INPUTS / 4]; + for (i, (_, value)) in node.inputs.iter().enumerate() { + inputs[i / 4][i % 4] = *value; + } + let (code, buffers, blend, bloom, zoom) = ( + node.code.clone(), + node.buffers.clone(), + node.blend, + node.bloom, + node.zoom, + ); + // Buffers evolve every frame, so a shader with any keeps running. + let moving = moving || !buffers.is_empty() && !cx.reduce_motion(); + let (local, time) = (self.local.clone(), self.time()); + let drag = orbit.drag; + el.child( + canvas( + move |bounds, window, _| { + if moving { + window.request_animation_frame(); + } + let scale = window.scale_factor(); + let size = ( + (f32::from(bounds.size.width) * scale).round().max(1.0) as u32, + (f32::from(bounds.size.height) * scale).round().max(1.0) as u32, + ); + let mut state = local.borrow_mut(); + // Shadertoy's mouse: pixels from the bottom-left; xy while + // pressed and zw where it went down (negative after). + let device = |p: Point| { + let at = p - bounds.origin; + [f32::from(at.x) * scale, size.1 as f32 - f32::from(at.y) * scale] + }; + let mouse = match (drag, state.pointers.get(&key)) { + (Some((start, _, _)), Some(now)) => { + let ([x, y], [sx, sy]) = (device(*now), device(start)); + [x, y, sx, sy] + } + (None, Some(now)) => { + let [x, y] = device(*now); + [x, y, -1.0, -1.0] + } + _ => [0.0, 0.0, -1.0, -1.0], + }; + let buffers_run = !buffers.is_empty(); + let previous = state.shaders.get(&key); + let shown_time = if animate { + time + } else { + previous.map_or(0.0, |p| p.last_time) + }; + let frame = previous.map_or(0, |p| p.frame + 1); + let delta = previous.map_or(0.0, |p| (time - p.last_time).max(0.0)); + let uniforms = zeron_visual::shader::ShaderUniforms { + resolution: [size.0 as f32, size.1 as f32, scale, 0.0], + mouse, + camera: [yaw, orbit.pitch, zoom, 0.0], + time: [shown_time as f32, delta as f32, frame as f32, 0.0], + inputs, + channels: Default::default(), + }; + let unchanged = !buffers_run && previous.is_some_and(|p| { + p.size == size + && p.uniforms.resolution == uniforms.resolution + && p.uniforms.mouse == uniforms.mouse + && p.uniforms.camera == uniforms.camera + && p.uniforms.inputs == uniforms.inputs + && (!animate || p.uniforms.time[0] == uniforms.time[0]) + }); + if unchanged { + return previous.map(|p| p.image.clone()); + } + let mut gpu_state = state.shaders.get_mut(&key).and_then(|p| p.gpu.take()); + let rendered = zeron_visual_gpu::render_shader( + &zeron_visual_gpu::ShaderSpec { + image: &code, + buffers: &buffers, + inputs: &names, + blend, + bloom, + }, + &uniforms, + size.0, + size.1, + &mut gpu_state, + ); + match rendered { + Ok(rendered) => { + let buffer = + image::RgbaImage::from_raw(rendered.width, rendered.height, rendered.bgra)?; + let image = + Arc::new(gpui::RenderImage::new([image::Frame::new(buffer)])); + let replaced = state.shaders.insert( + key.clone(), + ShaderCache { + uniforms, + size, + image: image.clone(), + frame, + last_time: time, + gpu: gpu_state, + }, + ); + if let Some(old) = replaced { + let _ = window.drop_image(old.image); + } + state.draw_error = None; + Some(image) + } + Err(error) => { + state.draw_error = Some(error); + None + } + } + }, + move |bounds, image, window, _| { + if let Some(image) = image { + let _ = window.paint_image( + bounds, + gpui::Corners::default(), + image, + 0, + false, + ); + } + }, + ) + .size_full(), + ) + .into_any_element() + } + fn scene( &mut self, node: &zeron_visual::SceneNode, @@ -1381,6 +1646,7 @@ impl VisualFrame { zoom: node.zoom, }; let moving = (spin != 0.0 && orbit.drag.is_none()) || node.animate; + let dragging = orbit.drag.is_some(); let (local, generation, time) = (self.local.clone(), self.generation, self.time()); let (draw, animate) = (node.draw, node.animate); let paint_palette = Rc::new(Palette::new(Theme::of(cx))); @@ -1390,68 +1656,14 @@ impl VisualFrame { .h(px(node.height)); el = self.pointer_tracking(el, &key, cx); if node.orbit { - let (down_key, drag_key, move_key, up_key, out_key) = ( - key.clone(), - key.clone(), - key.clone(), - key.clone(), - key.clone(), - ); - let release = move |this: &mut VisualFrame, key: &SharedString| { - if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(key) - && orbit.drag.take().is_some() - { - orbit.spin_epoch = Instant::now(); - } - }; - el = el - .cursor_grab() - .on_mouse_down( - MouseButton::Left, - cx.listener(move |this, event: &gpui::MouseDownEvent, _, cx| { - cx.stop_propagation(); - if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(&down_key) { - orbit.yaw += spin * orbit.spin_epoch.elapsed().as_secs_f32(); - orbit.drag = Some((event.position, orbit.yaw, orbit.pitch)); - } - cx.notify(); - }), - ) - .on_drag(VisualDrag(drag_key), |_, _, _, cx| cx.new(|_| DragGhost)) - .on_drag_move(cx.listener( - move |this, event: &gpui::DragMoveEvent, _, cx| { - if event.drag(cx).0 != move_key { - return; - } - if let Some(orbit) = this.local.borrow_mut().cameras.get_mut(&move_key) - && let Some((start, yaw, pitch)) = orbit.drag - { - let delta = event.event.position - start; - orbit.yaw = yaw - f32::from(delta.x) * 0.01; - orbit.pitch = (pitch + f32::from(delta.y) * 0.01).clamp(-1.5, 1.5); - } - cx.notify(); - }, - )) - .on_mouse_up( - MouseButton::Left, - cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { - release(this, &up_key); - cx.notify(); - }), - ) - .on_mouse_up_out( - MouseButton::Left, - cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { - release(this, &out_key); - cx.notify(); - }), - ); + el = self.orbit_handlers(el, &key, spin, cx); } let pointer_key = key.clone(); + let render_palette = paint_palette.clone(); el.child( canvas( move |bounds, window, _| { + let paint_palette = &render_palette; if moving { window.request_animation_frame(); } @@ -1470,13 +1682,26 @@ impl VisualFrame { match drawn { Ok(scene) => { let scene = Rc::new(scene); - state.scenes.insert( + // A new drawing replaces the old one's image. + let previous = state.scenes.insert( key.clone(), SceneCache { generation, scene: scene.clone(), + image: None, + final_cost: None, }, ); + if let Some(previous) = previous { + // An animated scene redraws every + // frame; its render cost carries over. + if let Some(entry) = state.scenes.get_mut(&key) { + entry.final_cost = previous.final_cost; + } + if let Some((_, old)) = previous.image { + let _ = window.drop_image(old.image); + } + } state.draw_error = None; scene } @@ -1487,15 +1712,85 @@ impl VisualFrame { } } }; - let projection = scene.project( - camera, - f32::from(bounds.size.width), - f32::from(bounds.size.height), - ); + // Always at full density. While the camera moves, a scene + // too heavy to supersample within a frame drops to one + // sample per pixel; at rest it is always supersampled. + let pixel_scale = window.scale_factor(); + let cheap = state + .scenes + .get(&key) + .and_then(|c| c.final_cost) + .is_some_and(|cost| cost < std::time::Duration::from_millis(8)); + let quality = if (moving || dragging) && !cheap { + zeron_visual::Quality::Draft + } else { + zeron_visual::Quality::Final + }; + let size = (f32::from(bounds.size.width), f32::from(bounds.size.height)); + let raster_key = RasterKey { + yaw: camera.yaw, + pitch: camera.pitch, + zoom: camera.zoom, + size, + pixel_scale, + quality, + palette: palette_key(paint_palette), + }; + let entry = state.scenes.get_mut(&key)?; + let image = match &entry.image { + Some((cached_key, image)) if *cached_key == raster_key => image.clone(), + _ => { + let palette = paint_palette.clone(); + let rgba = move |color| { + let rgba = palette.color(color).to_rgb(); + [rgba.r, rgba.g, rgba.b, rgba.a] + }; + // On the GPU when there is one (materials, shadows, + // glass, bloom); the software renderer otherwise. + let (width, height, bgra, hints, labels) = match zeron_visual_gpu::render_scene( + &scene, + camera, + size.0, + size.1, + pixel_scale, + &rgba, + ) { + Some(frame) => { + let (hints, labels) = scene.annotations(camera, size.0, size.1); + (frame.width, frame.height, frame.bgra, hints, labels) + } + None => { + let started = Instant::now(); + let raster = scene.render( + camera, + size.0, + size.1, + pixel_scale, + quality, + &rgba, + ); + if quality == zeron_visual::Quality::Final { + entry.final_cost = Some(started.elapsed()); + } + (raster.width, raster.height, raster.bgra, raster.hints, raster.labels) + } + }; + let buffer = image::RgbaImage::from_raw(width, height, bgra)?; + let image = SceneImage { + image: Arc::new(gpui::RenderImage::new([image::Frame::new(buffer)])), + hints: Rc::new(hints), + labels: Rc::new(labels), + }; + if let Some((_, old)) = entry.image.replace((raster_key, image.clone())) { + let _ = window.drop_image(old.image); + } + image + } + }; let hovered = state.pointers.get(&pointer_key).and_then(|p| { let at = *p - bounds.origin; let (x, y) = (f32::from(at.x), f32::from(at.y)); - projection + image .hints .iter() .map(|([hx, hy], r, text)| ((hx - x).hypot(hy - y), *r, text)) @@ -1503,13 +1798,33 @@ impl VisualFrame { .min_by(|a, b| a.0.total_cmp(&b.0)) .map(|(_, _, text)| text.clone()) }); - Some((projection, hovered)) + Some((image, hovered)) }, - move |bounds, projected, window, cx| { - let Some((projection, hovered)) = projected else { + move |bounds, rendered, window, cx| { + let Some((image, hovered)) = rendered else { return; }; - paint_scene(&projection, bounds.origin, &paint_palette, window, cx); + let _ = window.paint_image( + bounds, + gpui::Corners::default(), + image.image.clone(), + 0, + false, + ); + for label in image.labels.iter() { + paint_text( + &label.text, + at(bounds.origin, label.at), + label.size, + FontWeight::NORMAL, + false, + paint_palette.color(label.color), + Align::Center, + Align::End, + window, + cx, + ); + } if let Some(hint) = hovered { paint_text( &hint, @@ -1532,81 +1847,29 @@ impl VisualFrame { } } -fn paint_scene( - projection: &zeron_visual::Projection, - origin: Point, - palette: &Palette, - window: &mut Window, - cx: &mut App, -) { - for prim in &projection.prims { - match prim { - Prim::Face { - points, - color, - shade, - } => { - // Grow each face half a pixel from its center so neighbours - // overlap and anti-aliasing leaves no seams between them. - let n = points.len() as f32; - let (cx_, cy_) = points - .iter() - .fold((0.0, 0.0), |(x, y), p| (x + p[0] / n, y + p[1] / n)); - let grown: Vec> = points - .iter() - .map(|[x, y]| { - let (dx, dy) = (x - cx_, y - cy_); - let len = dx.hypot(dy).max(1e-3); - at(origin, [x + dx / len * 0.5, y + dy / len * 0.5]) - }) - .collect(); - fill_convex(&grown, shaded(palette.color(*color), *shade), window); - } - Prim::Line { a, b, color, width } => stroke_path( - [at(origin, *a), at(origin, *b)], - false, - *width, - None, - palette.color(*color), - window, - ), - Prim::Dot { - at: center, - radius, - color, - } => { - let points: Vec> = (0..16) - .map(|i| { - let a = std::f32::consts::TAU * i as f32 / 16.0; - at( - origin, - [center[0] + radius * a.cos(), center[1] + radius * a.sin()], - ) - }) - .collect(); - fill_convex(&points, palette.color(*color), window); - } - Prim::Label { - at: anchor, - text, - color, - size, - } => { - paint_text( - text, - at(origin, *anchor), - *size, - FontWeight::NORMAL, - false, - palette.color(*color), - Align::Center, - Align::End, - window, - cx, - ); - } +/// Identifies a palette's colors, so a theme change re-renders scenes. +fn palette_key(palette: &Palette) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for color in [ + palette.text, + palette.muted, + palette.faint, + palette.accent, + palette.on_accent, + palette.success, + palette.warning, + palette.danger, + palette.border, + ] + .iter() + .chain(palette.chart.iter()) + { + for channel in [color.h, color.s, color.l, color.a] { + channel.to_bits().hash(&mut hasher); } } + hasher.finish() } // ---- charts --------------------------------------------------------------- diff --git a/crates/visual-gpu/Cargo.toml b/crates/visual-gpu/Cargo.toml new file mode 100644 index 0000000000..5cbc6c9690 --- /dev/null +++ b/crates/visual-gpu/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "zeron-visual-gpu" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "GPU rendering for agent visuals: shaders and 3D scenes drawn offscreen with wgpu" + +[dependencies] +zeron-visual.workspace = true +# Already GPUI's renderer on Linux; here it draws offscreen on every desktop. +wgpu = { version = "29", features = ["glsl"] } +pollster = "0.4" +tracing.workspace = true diff --git a/crates/visual-gpu/examples/render_shader.rs b/crates/visual-gpu/examples/render_shader.rs new file mode 100644 index 0000000000..d399144b7f --- /dev/null +++ b/crates/visual-gpu/examples/render_shader.rs @@ -0,0 +1,71 @@ +//! Render a `ui.shader` pass headlessly, composited over a chat-like +//! background, to check how it will look: +//! +//! `cargo run -p zeron-visual-gpu --example render_shader -- shader.glsl out.ppm \ +//! [--size 1440x840] [--blend light|alpha|opaque] [--bloom 1] [--time 2] \ +//! [--input name=value ...] [--background r,g,b]` + +use zeron_visual::shader::{Blend, ShaderUniforms}; +use zeron_visual_gpu::{ShaderSpec, render_shader}; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let (source, out) = (&args[0], &args[1]); + let code = std::fs::read_to_string(source).expect("shader source"); + let (mut w, mut h, mut blend, mut bloom, mut time) = (1440u32, 840u32, Blend::Light, 0.0, 2.0); + let mut inputs: Vec<(String, f32)> = Vec::new(); + let mut background = [0.80f32, 0.83, 0.81]; + let mut i = 2; + while i < args.len() { + let value = args.get(i + 1).cloned().unwrap_or_default(); + match args[i].as_str() { + "--size" => { + let (a, b) = value.split_once('x').unwrap(); + (w, h) = (a.parse().unwrap(), b.parse().unwrap()); + } + "--blend" => blend = Blend::parse(&value).unwrap(), + "--bloom" => bloom = value.parse().unwrap(), + "--time" => time = value.parse().unwrap(), + "--input" => { + let (name, v) = value.split_once('=').unwrap(); + inputs.push((name.to_owned(), v.parse().unwrap())); + } + "--background" => { + let parts: Vec = value.split(',').map(|p| p.parse().unwrap()).collect(); + background = [parts[0], parts[1], parts[2]]; + } + other => panic!("unknown flag {other}"), + } + i += 2; + } + inputs.sort_by(|a, b| a.0.cmp(&b.0)); + let names: Vec = inputs.iter().map(|(n, _)| n.clone()).collect(); + let mut slots = [[0.0f32; 4]; 4]; + for (k, (_, v)) in inputs.iter().enumerate() { + slots[k / 4][k % 4] = *v; + } + let uniforms = ShaderUniforms { + resolution: [w as f32, h as f32, 2.0, 0.0], + mouse: [0.0, 0.0, -1.0, -1.0], + camera: [0.3, 0.45, 1.0, 0.0], + time: [time, 0.016, 60.0, 0.0], + inputs: slots, + channels: Default::default(), + }; + let spec = ShaderSpec { + image: &code, + buffers: &[], + inputs: &names, + blend, + bloom, + }; + let frame = render_shader(&spec, &uniforms, w, h, &mut None).expect("render"); + let mut ppm = format!("P6\n{w} {h}\n255\n").into_bytes(); + for p in frame.bgra.chunks_exact(4) { + let a = f32::from(p[3]) / 255.0; + for (ch, bg) in [p[2], p[1], p[0]].into_iter().zip(background) { + ppm.push((f32::from(ch) * a + bg * 255.0 * (1.0 - a)) as u8); + } + } + std::fs::write(out, ppm).expect("write"); +} diff --git a/crates/visual-gpu/src/lib.rs b/crates/visual-gpu/src/lib.rs new file mode 100644 index 0000000000..fac8aeca3b --- /dev/null +++ b/crates/visual-gpu/src/lib.rs @@ -0,0 +1,174 @@ +//! The GPU behind agent visuals. +//! +//! GPUI composites images, not foreign GPU passes, so visuals that need the +//! GPU (fragment shaders, lit 3D scenes) render offscreen here through wgpu +//! (Metal, Vulkan, or Direct3D) into a texture that is read back as +//! straight-alpha BGRA and painted like any other image, clipped, faded, and +//! layered with the transcript. One device serves every visual; pipelines are +//! cached by source, so a shader compiles once however many frames it draws. + +use std::sync::OnceLock; + +mod scene; +mod shader; + +pub use scene::render_scene; +pub use shader::{ShaderSpec, ShaderState, render_shader}; + +/// One rendered frame: straight-alpha BGRA, top row first. +#[derive(Debug, Clone)] +pub struct Frame { + pub width: u32, + pub height: u32, + pub bgra: Vec, +} + +pub(crate) struct Gpu { + pub(crate) device: wgpu::Device, + pub(crate) queue: wgpu::Queue, + pub(crate) shaders: OnceLock, + pub(crate) scene: OnceLock, +} + +/// The shared device, made on first use; `None` without a usable adapter. +pub(crate) fn gpu() -> Option<&'static Gpu> { + static GPU: OnceLock> = OnceLock::new(); + GPU.get_or_init(|| match pollster::block_on(Gpu::new()) { + Ok(gpu) => Some(gpu), + Err(error) => { + tracing::warn!(%error, "visual GPU unavailable"); + None + } + }) + .as_ref() +} + +/// Whether visuals can render on the GPU here. +pub fn available() -> bool { + gpu().is_some() +} + +/// The target every visual renders into before readback. +pub(crate) const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; + +impl Gpu { + async fn new() -> Result { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { + backends: wgpu::Backends::PRIMARY, + flags: wgpu::InstanceFlags::default(), + backend_options: wgpu::BackendOptions::default(), + memory_budget_thresholds: wgpu::MemoryBudgetThresholds::default(), + display: None, + }); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::LowPower, + force_fallback_adapter: false, + compatible_surface: None, + }) + .await + .map_err(|e| e.to_string())?; + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor { + label: Some("zeron visuals"), + ..Default::default() + }) + .await + .map_err(|e| e.to_string())?; + Ok(Self { + device, + queue, + shaders: OnceLock::new(), + scene: OnceLock::new(), + }) + } + +/// Read a rendered texture back as straight-alpha BGRA. + pub(crate) fn read_back( + &self, + mut encoder: wgpu::CommandEncoder, + texture: &wgpu::Texture, + opaque: bool, + ) -> Frame { + let (width, height) = (texture.width(), texture.height()); + let row = (width * 4).div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) + * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let buffer = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("visual readback"), + size: u64::from(row * height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + encoder.copy_texture_to_buffer( + texture.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(row), + rows_per_image: None, + }, + }, + texture.size(), + ); + self.queue.submit([encoder.finish()]); + let (tx, rx) = std::sync::mpsc::channel(); + buffer + .slice(..) + .map_async(wgpu::MapMode::Read, move |result| { + let _ = tx.send(result); + }); + let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); + let mut bgra = vec![0u8; (width * height * 4) as usize]; + if let Ok(Ok(())) = rx.recv() { + let mapped = buffer.slice(..).get_mapped_range(); + for y in 0..height as usize { + let src = &mapped[y * row as usize..][..width as usize * 4]; + let dst = &mut bgra[y * width as usize * 4..][..width as usize * 4]; + for (d, s) in dst.chunks_exact_mut(4).zip(src.chunks_exact(4)) { + d[0] = s[2]; + d[1] = s[1]; + d[2] = s[0]; + d[3] = if opaque { 255 } else { s[3] }; + } + } + drop(mapped); + buffer.unmap(); + } + Frame { + width, + height, + bgra, + } + } + + pub(crate) fn target(&self, width: u32, height: u32, label: &str) -> wgpu::Texture { + self.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }) + } +} + +#[cfg(test)] +pub(crate) fn dump_ppm(frame: &Frame, path: &str) { + let mut out = format!("P6\n{} {}\n255\n", frame.width, frame.height).into_bytes(); + for p in frame.bgra.chunks_exact(4) { + // Over a mid grey, so transparency reads. + let a = f32::from(p[3]) / 255.0; + for ch in [p[2], p[1], p[0]] { + out.push((f32::from(ch) * a + 60.0 * (1.0 - a)) as u8); + } + } + let _ = std::fs::write(path, out); +} diff --git a/crates/visual-gpu/src/scene.rs b/crates/visual-gpu/src/scene.rs new file mode 100644 index 0000000000..cc4e512b5c --- /dev/null +++ b/crates/visual-gpu/src/scene.rs @@ -0,0 +1,1242 @@ +//! `ui.scene` on the GPU: physically based shading, soft shadows, glass that +//! refracts and splits light, and bloom, rendered offscreen and read back. +//! +//! Passes, per frame: +//! 1. shadow: the key light's depth over the geometry (orthographic); +//! 2. main: opaque surfaces into a multisampled HDR target with depth (GGX +//! specular, sky/ground ambient, key light with PCF shadows, a fill +//! light, glow), then lines and points unlit; resolved for sampling; +//! 3. glass: transmissive surfaces over that, refracting the resolved image +//! per color channel (dispersion) under a Fresnel reflection; +//! 4. bloom: bright parts downsampled and blurred; +//! 5. composite: bloom added, a fine depth-edge outline, tonemapped to sRGB +//! straight alpha, transparent wherever nothing was drawn. + +use wgpu::util::DeviceExt; +use zeron_visual::{Camera, CameraFrame, Color, Scene, V3}; + +use crate::{FORMAT, Frame, gpu}; + +const HDR: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; +const DEPTH: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float; +const SAMPLES: u32 = 4; +const SHADOW_SIZE: u32 = 2048; + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct Vertex { + pos: [f32; 3], + normal: [f32; 3], + /// The face's own normal, for culling closed solids. + face: [f32; 3], + /// Linear color, straight alpha. + color: [f32; 4], + /// Roughness, metalness, glow, glass. + mat: [f32; 4], + /// Ior, dispersion, two-sided, kind (0 surface, 1 line, 2 point disc). + opt: [f32; 4], + /// Point-disc coordinates in -1..1. + uv: [f32; 2], +} + +impl Vertex { + const ATTRIBUTES: [wgpu::VertexAttribute; 7] = wgpu::vertex_attr_array![ + 0 => Float32x3, 1 => Float32x3, 2 => Float32x3, 3 => Float32x4, + 4 => Float32x4, 5 => Float32x4, 6 => Float32x2 + ]; + + fn layout() -> wgpu::VertexBufferLayout<'static> { + wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &Self::ATTRIBUTES, + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct Globals { + view_proj: [[f32; 4]; 4], + light_view_proj: [[f32; 4]; 4], + eye: [f32; 4], + /// Toward the key light. + light: [f32; 4], + sky: [f32; 4], + ground: [f32; 4], + /// Width, height, 1/width, 1/height (device pixels). + viewport: [f32; 4], + /// Near, far, the geometry's radius, unused. + depth: [f32; 4], + /// The camera's right, up, and forward, for screen-space refraction. + right: [f32; 4], + up: [f32; 4], + forward: [f32; 4], +} + +fn bytes(value: &T) -> &[u8] { + // SAFETY: plain-old-data `repr(C)` structs of `f32`s. + unsafe { + std::slice::from_raw_parts((value as *const T).cast::(), std::mem::size_of::()) + } +} + +fn slice_bytes(values: &[T]) -> &[u8] { + // SAFETY: as above, for a slice. + unsafe { + std::slice::from_raw_parts(values.as_ptr().cast::(), std::mem::size_of_val(values)) + } +} + +pub(crate) struct Pipelines { + globals_layout: wgpu::BindGroupLayout, + glass_layout: wgpu::BindGroupLayout, + post_layout: wgpu::BindGroupLayout, + shadow: wgpu::RenderPipeline, + opaque: wgpu::RenderPipeline, + overlay: wgpu::RenderPipeline, + glass: wgpu::RenderPipeline, + bright: wgpu::RenderPipeline, + blur: wgpu::RenderPipeline, + composite: wgpu::RenderPipeline, + linear: wgpu::Sampler, + shadow_sampler: wgpu::Sampler, +} + +const SURFACE: &str = r#" +struct Globals { + view_proj: mat4x4, + light_view_proj: mat4x4, + eye: vec4, + light: vec4, + sky: vec4, + ground: vec4, + viewport: vec4, + depth: vec4, + right: vec4, + up: vec4, + forward: vec4, +}; +@group(0) @binding(0) var g: Globals; +@group(0) @binding(1) var shadow_map: texture_depth_2d; +@group(0) @binding(2) var shadow_sampler: sampler_comparison; + +struct In { + @location(0) pos: vec3, + @location(1) normal: vec3, + @location(2) face: vec3, + @location(3) color: vec4, + @location(4) mat: vec4, + @location(5) opt: vec4, + @location(6) uv: vec2, +}; +struct Out { + @builtin(position) clip: vec4, + @location(0) pos: vec3, + @location(1) normal: vec3, + @location(2) face: vec3, + @location(3) color: vec4, + @location(4) mat: vec4, + @location(5) opt: vec4, + @location(6) uv: vec2, +}; + +@vertex fn vs(v: In) -> Out { + var o: Out; + o.clip = g.view_proj * vec4(v.pos, 1.0); + o.pos = v.pos; + o.normal = v.normal; + o.face = v.face; + o.color = v.color; + o.mat = v.mat; + o.opt = v.opt; + o.uv = v.uv; + return o; +} + +@vertex fn vs_shadow(v: In) -> @builtin(position) vec4 { + return g.light_view_proj * vec4(v.pos, 1.0); +} + +const PI: f32 = 3.14159265; + +fn shadow(pos: vec3, n: vec3) -> f32 { + let clip = g.light_view_proj * vec4(pos + n * 0.004 * g.depth.z, 1.0); + let uv = vec2(clip.x * 0.5 + 0.5, 0.5 - clip.y * 0.5); + if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) { return 1.0; } + let texel = 1.0 / f32(textureDimensions(shadow_map).x); + var lit = 0.0; + for (var y = -1; y <= 1; y++) { + for (var x = -1; x <= 1; x++) { + lit += textureSampleCompareLevel(shadow_map, shadow_sampler, + uv + vec2(f32(x), f32(y)) * texel * 1.5, clip.z - 0.0015); + } + } + return lit / 9.0; +} + +fn ggx(n_h: f32, rough: f32) -> f32 { + let a = rough * rough; + let a2 = a * a; + let d = n_h * n_h * (a2 - 1.0) + 1.0; + return a2 / (PI * d * d); +} + +fn smith(n_v: f32, n_l: f32, rough: f32) -> f32 { + let k = (rough + 1.0) * (rough + 1.0) / 8.0; + return (n_v / (n_v * (1.0 - k) + k)) * (n_l / (n_l * (1.0 - k) + k)); +} + +fn fresnel(cos_t: f32, f0: vec3) -> vec3 { + return f0 + (1.0 - f0) * pow(1.0 - cos_t, 5.0); +} + +fn environment(dir: vec3) -> vec3 { + return mix(g.ground.rgb, g.sky.rgb, smoothstep(-0.3, 0.6, dir.y)); +} + +// The shading normal turned toward the camera; w is 0 on the far side of a +// closed solid, which is not drawn. +fn facing(i: Out, v: vec3) -> vec4 { + var n = normalize(i.normal); + if (i.opt.z < 0.5 && dot(i.face, v) < 0.0) { return vec4(0.0); } + if (dot(n, v) < 0.0) { n = -n; } + return vec4(n, 1.0); +} + +fn shade(i: Out, n: vec3, v: vec3) -> vec3 { + let albedo = i.color.rgb; + let rough = clamp(i.mat.x, 0.04, 1.0); + let metal = i.mat.y; + let l = normalize(g.light.xyz); + let h = normalize(l + v); + let n_l = max(dot(n, l), 0.0); + let n_v = max(dot(n, v), 0.001); + let n_h = max(dot(n, h), 0.0); + let f0 = mix(vec3(0.04), albedo, metal); + let f = fresnel(max(dot(h, v), 0.0), f0); + let spec = ggx(n_h, rough) * smith(n_v, n_l, rough) * f / max(4.0 * n_v * n_l, 0.001); + let kd = (vec3(1.0) - f) * (1.0 - metal); + let key = (kd * albedo / PI + spec) * n_l * 3.2 * shadow(i.pos, n); + let fill_dir = normalize(vec3(-l.x, 0.35, -l.z)); + let fill = kd * albedo / PI * max(dot(n, fill_dir), 0.0) * 0.9; + let r = reflect(-v, n); + let env_f = fresnel(n_v, f0); + let ambient = kd * albedo * environment(n) * 0.85 + + env_f * environment(r) * (1.0 - rough * 0.85) * 0.8; + return key + fill + ambient + albedo * i.mat.z; +} + +@fragment fn fs_opaque(i: Out) -> @location(0) vec4 { + let v = normalize(g.eye.xyz - i.pos); + let f = facing(i, v); + if (f.w == 0.0) { discard; } + return vec4(shade(i, f.xyz, v), 1.0); +} + +@fragment fn fs_overlay(i: Out) -> @location(0) vec4 { + if (i.opt.w > 1.5 && dot(i.uv, i.uv) > 1.0) { discard; } + let c = i.color.rgb * (1.0 + i.mat.z); + return vec4(c * i.color.a, i.color.a); +} +"#; + +const GLASS: &str = r#" +@group(1) @binding(0) var behind: texture_2d; +@group(1) @binding(1) var behind_sampler: sampler; + +@fragment fn fs_glass(i: Out) -> @location(0) vec4 { + let v = normalize(g.eye.xyz - i.pos); + let f = facing(i, v); + if (f.w == 0.0) { discard; } + let n = f.xyz; + let ior = max(i.opt.x, 1.0); + let spread = i.opt.y * 0.09; + let screen = i.clip.xy * g.viewport.zw; + // Bend what's behind by the surface's tilt across the view, a little + // more for each shorter wavelength. + let view_n = vec2(dot(n, g.right.xyz), -dot(n, g.up.xyz)); + var seen = vec3(0.0); + var cover = 0.0; + let iors = vec3(ior - spread, ior, ior + spread); + for (var c = 0; c < 3; c++) { + let bend = (1.0 - 1.0 / iors[c]) * 0.34; + let s = textureSampleLevel(behind, behind_sampler, screen - view_n * bend, 0.0); + seen[c] = s[c]; + cover = max(cover, s.a); + } + let tint = mix(vec3(1.0), i.color.rgb, 0.35); + let f0 = pow((ior - 1.0) / (ior + 1.0), 2.0); + let n_v = max(dot(n, v), 0.0); + let fr = f0 + (1.0 - f0) * pow(1.0 - n_v, 5.0); + let r = reflect(-v, n); + let l = normalize(g.light.xyz); + let glint = pow(max(dot(r, l), 0.0), 220.0) * 6.0; + let reflection = environment(r) * 1.2 + vec3(glint); + let through = seen * tint; + let glass = clamp(i.mat.w, 0.0, 1.0); + // Where nothing stands behind the glass, it shows its reflection and a + // faint body tint over the chat. + let body = 0.12 + 0.25 * (1.0 - glass); + let alpha = clamp(max(cover, fr + body + glint), 0.0, 1.0); + let color = through + (reflection - through) * fr + i.color.rgb * body * (1.0 - cover) + + i.color.rgb * i.mat.z; + return vec4(color, alpha); +} +"#; + +const POST: &str = r#" +@group(0) @binding(0) var source: texture_2d; +@group(0) @binding(1) var source_sampler: sampler; +@group(0) @binding(2) var post: vec4; +@group(0) @binding(3) var bloom: texture_2d; +@group(0) @binding(4) var depth: texture_depth_multisampled_2d; + +struct Out { @builtin(position) clip: vec4, @location(0) uv: vec2 }; + +@vertex fn vs(@builtin(vertex_index) i: u32) -> Out { + let p = vec2(f32((i << 1u) & 2u), f32(i & 2u)); + var o: Out; + o.clip = vec4(p * 2.0 - 1.0, 0.0, 1.0); + o.uv = vec2(p.x, 1.0 - p.y); + return o; +} + +fn luma(c: vec3) -> f32 { return dot(c, vec3(0.2126, 0.7152, 0.0722)); } + +@fragment fn fs_bright(i: Out) -> @location(0) vec4 { + let c = textureSampleLevel(source, source_sampler, i.uv, 0.0); + let over = max(luma(c.rgb) - 1.0, 0.0); + return vec4(c.rgb * (over / max(luma(c.rgb), 0.0001)), 1.0); +} + +// post.xy: one texel step along the blur direction. +@fragment fn fs_blur(i: Out) -> @location(0) vec4 { + let w = array(0.227, 0.194, 0.121, 0.054, 0.016); + var sum = textureSampleLevel(source, source_sampler, i.uv, 0.0).rgb * w[0]; + for (var k = 1; k < 5; k++) { + let o = post.xy * f32(k) * 1.5; + sum += textureSampleLevel(source, source_sampler, i.uv + o, 0.0).rgb * w[k]; + sum += textureSampleLevel(source, source_sampler, i.uv - o, 0.0).rgb * w[k]; + } + return vec4(sum, 1.0); +} + +fn aces(x: vec3) -> vec3 { + let a = 2.51; let b = 0.03; let c = 2.43; let d = 0.59; let e = 0.14; + return clamp((x * (a * x + b)) / (x * (c * x + d) + e), vec3(0.0), vec3(1.0)); +} + +fn to_srgb(c: vec3) -> vec3 { + let lo = c * 12.92; + let hi = 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055; + return select(hi, lo, c <= vec3(0.0031308)); +} + +const NEIGHBOURS = array, 4>(vec2(1, 0), vec2(-1, 0), vec2(0, 1), vec2(0, -1)); + +// post.x: bloom strength. +@fragment fn fs_composite(i: Out) -> @location(0) vec4 { + let c = textureSampleLevel(source, source_sampler, i.uv, 0.0); + let glow = textureSampleLevel(bloom, source_sampler, i.uv, 0.0).rgb * post.x; + // A fine outline where depth jumps: silhouettes and folds. + let size = vec2(textureDimensions(depth)); + let at = clamp(vec2(i.clip.xy), vec2(0), size - 1); + let z = textureLoad(depth, at, 0); + var edge = 0.0; + if (z < 1.0) { + for (var k = 0; k < 4; k++) { + let zn = textureLoad(depth, clamp(at + NEIGHBOURS[k], vec2(0), size - 1), 0); + if (zn >= 1.0 || abs(zn - z) > 0.004 * (1.0 - z) + 0.0004) { edge = 1.0; } + } + } + let alpha = clamp(c.a + luma(glow), 0.0, 1.0); + if (alpha <= 0.0001) { return vec4(0.0); } + var straight = (c.rgb + glow) / alpha; + straight = straight * (1.0 - 0.18 * edge); + return vec4(to_srgb(aces(straight)), alpha); +} +"#; + +impl Pipelines { + fn new(device: &wgpu::Device) -> Self { + let uniform = |binding, visibility| wgpu::BindGroupLayoutEntry { + binding, + visibility, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }; + let texture = |binding, sample_type, multisampled| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled, + }, + count: None, + }; + let sampler = |binding, kind| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(kind), + count: None, + }; + let float = wgpu::TextureSampleType::Float { filterable: true }; + let globals_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("scene globals"), + entries: &[ + uniform(0, wgpu::ShaderStages::VERTEX_FRAGMENT), + texture(1, wgpu::TextureSampleType::Depth, false), + sampler(2, wgpu::SamplerBindingType::Comparison), + ], + }); + let glass_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("scene behind glass"), + entries: &[ + texture(0, float, false), + sampler(1, wgpu::SamplerBindingType::Filtering), + ], + }); + let post_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("scene post"), + entries: &[ + texture(0, float, false), + sampler(1, wgpu::SamplerBindingType::Filtering), + uniform(2, wgpu::ShaderStages::FRAGMENT), + texture(3, float, false), + texture(4, wgpu::TextureSampleType::Depth, true), + ], + }); + let surface = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("scene surfaces"), + source: wgpu::ShaderSource::Wgsl(SURFACE.into()), + }); + let glass_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("scene glass"), + source: wgpu::ShaderSource::Wgsl(format!("{SURFACE}{GLASS}").into()), + }); + let post = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("scene post"), + source: wgpu::ShaderSource::Wgsl(POST.into()), + }); + let layout = |layouts: &[Option<&wgpu::BindGroupLayout>]| { + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: None, + bind_group_layouts: layouts, + immediate_size: 0, + }) + }; + let surface_layout = layout(&[Some(&globals_layout)]); + let glass_pipeline_layout = layout(&[Some(&globals_layout), Some(&glass_layout)]); + let post_pipeline_layout = layout(&[Some(&post_layout)]); + let msaa = wgpu::MultisampleState { + count: SAMPLES, + ..Default::default() + }; + let depth = |write: bool, bias: wgpu::DepthBiasState| { + Some(wgpu::DepthStencilState { + format: DEPTH, + depth_write_enabled: Some(write), + depth_compare: Some(wgpu::CompareFunction::Less), + stencil: Default::default(), + bias, + }) + }; + let surface_pipeline = |label: &str, + module: &wgpu::ShaderModule, + layout: &wgpu::PipelineLayout, + fragment: &str, + blend: Option, + depth_stencil: Option| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(layout), + vertex: wgpu::VertexState { + module, + entry_point: Some("vs"), + compilation_options: Default::default(), + buffers: &[Vertex::layout()], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil, + multisample: msaa, + fragment: Some(wgpu::FragmentState { + module, + entry_point: Some(fragment), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: HDR, + blend, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }; + let opaque = surface_pipeline( + "scene opaque", + &surface, + &surface_layout, + "fs_opaque", + None, + depth(true, Default::default()), + ); + let overlay = surface_pipeline( + "scene lines and points", + &surface, + &surface_layout, + "fs_overlay", + Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + depth( + false, + wgpu::DepthBiasState { + constant: -4, + slope_scale: -1.0, + clamp: 0.0, + }, + ), + ); + let glass = surface_pipeline( + "scene glass", + &glass_module, + &glass_pipeline_layout, + "fs_glass", + None, + depth(true, Default::default()), + ); + let shadow = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("scene shadow"), + layout: Some(&surface_layout), + vertex: wgpu::VertexState { + module: &surface, + entry_point: Some("vs_shadow"), + compilation_options: Default::default(), + buffers: &[Vertex::layout()], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: Some(wgpu::DepthStencilState { + format: DEPTH, + depth_write_enabled: Some(true), + depth_compare: Some(wgpu::CompareFunction::Less), + stencil: Default::default(), + bias: wgpu::DepthBiasState { + constant: 2, + slope_scale: 2.0, + clamp: 0.0, + }, + }), + multisample: Default::default(), + fragment: None, + multiview_mask: None, + cache: None, + }); + let post_pipeline = |label: &str, entry: &str, format| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&post_pipeline_layout), + vertex: wgpu::VertexState { + module: &post, + entry_point: Some("vs"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &post, + entry_point: Some(entry), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }; + Self { + bright: post_pipeline("scene bloom bright", "fs_bright", HDR), + blur: post_pipeline("scene bloom blur", "fs_blur", HDR), + composite: post_pipeline("scene composite", "fs_composite", FORMAT), + globals_layout, + glass_layout, + post_layout, + shadow, + opaque, + overlay, + glass, + linear: device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("scene linear"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + ..Default::default() + }), + shadow_sampler: device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("scene shadow"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + compare: Some(wgpu::CompareFunction::LessEqual), + ..Default::default() + }), + } + } +} + +fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +fn scale(a: V3, s: f32) -> V3 { + [a[0] * s, a[1] * s, a[2] * s] +} +fn dot(a: V3, b: V3) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +fn cross(a: V3, b: V3) -> V3 { + [ + a[1] * b[2] - a[2] * b[1], + a[2] * b[0] - a[0] * b[2], + a[0] * b[1] - a[1] * b[0], + ] +} +fn normalize(a: V3) -> V3 { + let len = dot(a, a).sqrt(); + if len < 1e-9 { + [0.0, 1.0, 0.0] + } else { + scale(a, 1.0 / len) + } +} + +fn to_linear(c: f32) -> f32 { + if c <= 0.04045 { + c / 12.92 + } else { + ((c + 0.055) / 1.055).powf(2.4) + } +} + +fn linear(rgba: [f32; 4]) -> [f32; 4] { + [ + to_linear(rgba[0]), + to_linear(rgba[1]), + to_linear(rgba[2]), + rgba[3], + ] +} + +/// Rows of a matrix into WGSL's column-major layout. +fn columns(rows: [[f32; 4]; 4]) -> [[f32; 4]; 4] { + let mut cols = [[0.0; 4]; 4]; + for (i, row) in rows.iter().enumerate() { + for (j, value) in row.iter().enumerate() { + cols[j][i] = *value; + } + } + cols +} + +/// The world-space key light, from above and to the side, so shadows fall +/// toward the viewer's default angle and ground the geometry. +const KEY_LIGHT: V3 = [-0.45, 0.85, 0.35]; + +struct Geometry { + /// Opaque surfaces; the first `shadowed` vertices cast shadows (glowing + /// surfaces, which are lights, come after and don't). + opaque: Vec, + shadowed: usize, + glass: Vec, + overlay: Vec, +} + +fn geometry(scene: &Scene, frame: &CameraFrame, rgba: &dyn Fn(Color) -> [f32; 4]) -> Geometry { + let mut glowing = Vec::new(); + let mut out = Geometry { + opaque: Vec::new(), + shadowed: 0, + glass: Vec::new(), + overlay: Vec::new(), + }; + let mut glass_faces: Vec<(f32, Vec)> = Vec::new(); + for face in scene.face_views() { + let m = face.material; + let color = linear(rgba(face.color)); + let vertex = |i: usize| Vertex { + pos: face.corners[i], + normal: face.normals.map_or(face.normal, |n| n[i]), + face: face.normal, + color, + mat: [m.roughness, m.metalness, m.glow, m.glass], + opt: [ + m.ior, + m.dispersion, + if face.two_sided { 1.0 } else { 0.0 }, + 0.0, + ], + uv: [0.0; 2], + }; + let mut tris = Vec::with_capacity((face.corners.len() - 2) * 3); + for i in 1..face.corners.len() - 1 { + tris.extend([vertex(0), vertex(i), vertex(i + 1)]); + } + if m.glass > 0.0 || color[3] < 0.999 { + let centroid = face.corners.iter().fold([0.0; 3], |acc, c| add(acc, *c)); + let centroid = scale(centroid, 1.0 / face.corners.len() as f32); + glass_faces.push((dot(sub(centroid, frame.eye), frame.forward), tris)); + } else if m.glow > 0.0 { + glowing.extend(tris); + } else { + out.opaque.extend(tris); + } + } + out.shadowed = out.opaque.len(); + out.opaque.extend(glowing); + // Glass far to near, so nearer panes cover farther ones. + glass_faces.sort_by(|a, b| b.0.total_cmp(&a.0)); + out.glass = glass_faces.into_iter().flat_map(|(_, t)| t).collect(); + + // Lines and points face the camera, sized in screen pixels. + let world_per_pixel = + |p: V3| dot(sub(p, frame.eye), frame.forward).max(frame.near) / frame.focal; + let flat = |pos: V3, color: [f32; 4], kind: f32, uv: [f32; 2]| Vertex { + pos, + normal: [0.0, 1.0, 0.0], + face: [0.0, 1.0, 0.0], + color, + mat: [1.0, 0.0, 0.0, 0.0], + opt: [1.0, 0.0, 1.0, kind], + uv, + }; + for (a, b, color, width) in scene.line_views() { + let color = linear(rgba(color)); + let dir = normalize(sub(b, a)); + let side_a = normalize(cross(dir, sub(a, frame.eye))); + let side_b = normalize(cross(dir, sub(b, frame.eye))); + let wa = scale(side_a, width.max(1.0) * 0.5 * world_per_pixel(a)); + let wb = scale(side_b, width.max(1.0) * 0.5 * world_per_pixel(b)); + let quad = [add(a, wa), add(b, wb), sub(b, wb), sub(a, wa)]; + for i in [0, 1, 2, 0, 2, 3] { + out.overlay.push(flat(quad[i], color, 1.0, [0.0; 2])); + } + } + for (at, radius, color) in scene.point_views() { + let color = linear(rgba(color)); + let r = radius * world_per_pixel(at); + let (x, y) = (scale(frame.right, r), scale(frame.up, r)); + let corners = [ + (add(sub(at, x), y), [-1.0, 1.0]), + (add(add(at, x), y), [1.0, 1.0]), + (sub(add(at, x), y), [1.0, -1.0]), + (sub(sub(at, x), y), [-1.0, -1.0]), + ]; + for i in [0, 1, 2, 0, 2, 3] { + out.overlay.push(flat(corners[i].0, color, 2.0, corners[i].1)); + } + } + out +} + +/// Render `scene` on the GPU for a canvas `width` by `height` logical pixels +/// at `pixel_scale`. `None` without a GPU (or geometry), so the caller can +/// fall back to the software renderer. +pub fn render_scene( + scene: &Scene, + camera: Camera, + width: f32, + height: f32, + pixel_scale: f32, + rgba: &dyn Fn(Color) -> [f32; 4], +) -> Option { + let gpu = gpu()?; + let frame = scene.camera_frame(camera, width, height)?; + let (w, h) = ( + (width * pixel_scale).round().max(1.0) as u32, + (height * pixel_scale).round().max(1.0) as u32, + ); + let pipelines = gpu.scene.get_or_init(|| Pipelines::new(&gpu.device)); + let device = &gpu.device; + let geometry = geometry(scene, &frame, rgba); + + // Camera: the same framing and projection as the software renderer. + let distance = dot(sub(frame.center, frame.eye), sub(frame.center, frame.eye)).sqrt(); + let near = frame.near; + let far = distance + frame.radius * 2.0; + let (sx, sy) = (frame.focal / (width / 2.0), frame.focal / (height / 2.0)); + let (a, b) = (far / (far - near), -far * near / (far - near)); + let row = |axis: V3, s: f32| { + [ + axis[0] * s, + axis[1] * s, + axis[2] * s, + -dot(axis, frame.eye) * s, + ] + }; + let f = frame.forward; + let view_proj = columns([ + row(frame.right, sx), + row(frame.up, sy), + [f[0] * a, f[1] * a, f[2] * a, -dot(f, frame.eye) * a + b], + row(f, 1.0), + ]); + // The key light's orthographic view, fitted to the geometry. + let light = normalize(KEY_LIGHT); + let light_forward = scale(light, -1.0); + let light_right = normalize(cross(light_forward, [0.0, 0.0, 1.0])); + let light_up = cross(light_right, light_forward); + let r = frame.radius * 1.1; + let c = frame.center; + let light_view_proj = columns([ + [ + light_right[0] / r, + light_right[1] / r, + light_right[2] / r, + -dot(light_right, c) / r, + ], + [ + light_up[0] / r, + light_up[1] / r, + light_up[2] / r, + -dot(light_up, c) / r, + ], + [ + light_forward[0] / (2.0 * r), + light_forward[1] / (2.0 * r), + light_forward[2] / (2.0 * r), + (-dot(light_forward, c) + r) / (2.0 * r), + ], + [0.0, 0.0, 0.0, 1.0], + ]); + let globals = Globals { + view_proj, + light_view_proj, + eye: [frame.eye[0], frame.eye[1], frame.eye[2], 0.0], + light: [light[0], light[1], light[2], 0.0], + sky: [0.62, 0.66, 0.74, 0.0], + ground: [0.26, 0.24, 0.22, 0.0], + viewport: [w as f32, h as f32, 1.0 / w as f32, 1.0 / h as f32], + depth: [near, far, frame.radius, 0.0], + right: [frame.right[0], frame.right[1], frame.right[2], 0.0], + up: [frame.up[0], frame.up[1], frame.up[2], 0.0], + forward: [f[0], f[1], f[2], 0.0], + }; + let globals_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("scene globals"), + contents: bytes(&globals), + usage: wgpu::BufferUsages::UNIFORM, + }); + + // Targets. + use wgpu::TextureUsages as U; + let texture = |label: &str, size: (u32, u32), format, samples, usage| { + device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: size.0, + height: size.1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: samples, + dimension: wgpu::TextureDimension::D2, + format, + usage, + view_formats: &[], + }) + }; + let sampled = U::RENDER_ATTACHMENT | U::TEXTURE_BINDING; + let shadow_map = texture( + "scene shadow map", + (SHADOW_SIZE, SHADOW_SIZE), + DEPTH, + 1, + sampled, + ); + let msaa_color = texture("scene color", (w, h), HDR, SAMPLES, U::RENDER_ATTACHMENT); + let msaa_depth = texture("scene depth", (w, h), DEPTH, SAMPLES, sampled); + let opaque_resolved = texture("scene behind glass", (w, h), HDR, 1, sampled); + let lit = texture("scene lit", (w, h), HDR, 1, sampled); + let half = ((w / 2).max(1), (h / 2).max(1)); + let bloom_a = texture("scene bloom", half, HDR, 1, sampled); + let bloom_b = texture("scene bloom", half, HDR, 1, sampled); + let output = gpu.target(w, h, "scene output"); + let view = |t: &wgpu::Texture| t.create_view(&Default::default()); + + let globals_bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scene globals"), + layout: &pipelines.globals_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: globals_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&view(&shadow_map)), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&pipelines.shadow_sampler), + }, + ], + }); + let vertices = |label: &str, data: &[Vertex]| { + (!data.is_empty()).then(|| { + device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some(label), + contents: slice_bytes(data), + usage: wgpu::BufferUsages::VERTEX, + }) + }) + }; + let opaque = vertices("scene opaque", &geometry.opaque); + let glass = vertices("scene glass", &geometry.glass); + let overlay = vertices("scene overlay", &geometry.overlay); + let draw = |pass: &mut wgpu::RenderPass, buffer: &Option, count: usize| { + if let Some(buffer) = buffer { + pass.set_vertex_buffer(0, buffer.slice(..)); + pass.draw(0..count as u32, 0..1); + } + }; + + let mut encoder = device.create_command_encoder(&Default::default()); + // 1. Shadow. Its bind group (with the sampled map) is unused here: the + // shadow vertex stage only reads the globals. + let shadow_globals = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scene shadow globals"), + layout: &pipelines.globals_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: globals_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&view(&lit_dummy_depth(device))), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&pipelines.shadow_sampler), + }, + ], + }); + { + let shadow_view = view(&shadow_map); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("scene shadow"), + color_attachments: &[], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &shadow_view, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + store: wgpu::StoreOp::Store, + }), + stencil_ops: None, + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(&pipelines.shadow); + pass.set_bind_group(0, &shadow_globals, &[]); + draw(&mut pass, &opaque, geometry.shadowed); + } + let color_view = view(&msaa_color); + let depth_view = view(&msaa_depth); + // 2. Opaque surfaces, then lines and points. + { + let resolve = view(&opaque_resolved); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("scene main"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &color_view, + depth_slice: None, + resolve_target: Some(&resolve), + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &depth_view, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + store: wgpu::StoreOp::Store, + }), + stencil_ops: None, + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_bind_group(0, &globals_bind, &[]); + pass.set_pipeline(&pipelines.opaque); + draw(&mut pass, &opaque, geometry.opaque.len()); + pass.set_pipeline(&pipelines.overlay); + draw(&mut pass, &overlay, geometry.overlay.len()); + } + // 3. Glass, refracting the resolved opaque image. + let shaded = if glass.is_some() { + let behind_bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scene behind glass"), + layout: &pipelines.glass_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view(&opaque_resolved)), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&pipelines.linear), + }, + ], + }); + let resolve = view(&lit); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("scene glass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &color_view, + depth_slice: None, + resolve_target: Some(&resolve), + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &depth_view, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }), + stencil_ops: None, + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_bind_group(0, &globals_bind, &[]); + pass.set_bind_group(1, &behind_bind, &[]); + pass.set_pipeline(&pipelines.glass); + draw(&mut pass, &glass, geometry.glass.len()); + &lit + } else { + &opaque_resolved + }; + + // 4–5. Bloom and composite. + let post_uniform = |values: [f32; 4]| { + device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("scene post"), + contents: slice_bytes(&values), + usage: wgpu::BufferUsages::UNIFORM, + }) + }; + let depth_sample_view = msaa_depth.create_view(&wgpu::TextureViewDescriptor { + aspect: wgpu::TextureAspect::DepthOnly, + ..Default::default() + }); + let post_bind = |source: &wgpu::Texture, uniform: &wgpu::Buffer, bloom: &wgpu::Texture| { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scene post"), + layout: &pipelines.post_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view(source)), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&pipelines.linear), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(&view(bloom)), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: wgpu::BindingResource::TextureView(&depth_sample_view), + }, + ], + }) + }; + let fullscreen = |encoder: &mut wgpu::CommandEncoder, + target: &wgpu::Texture, + pipeline: &wgpu::RenderPipeline, + bind: &wgpu::BindGroup| { + let target = view(target); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("scene post"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &target, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(pipeline); + pass.set_bind_group(0, bind, &[]); + pass.draw(0..3, 0..1); + }; + let zero = post_uniform([0.0; 4]); + let step_x = post_uniform([1.0 / half.0 as f32, 0.0, 0.0, 0.0]); + let step_y = post_uniform([0.0, 1.0 / half.1 as f32, 0.0, 0.0]); + fullscreen( + &mut encoder, + &bloom_a, + &pipelines.bright, + &post_bind(shaded, &zero, &bloom_b), + ); + for _ in 0..2 { + fullscreen( + &mut encoder, + &bloom_b, + &pipelines.blur, + &post_bind(&bloom_a, &step_x, &lit_dummy(device)), + ); + fullscreen( + &mut encoder, + &bloom_a, + &pipelines.blur, + &post_bind(&bloom_b, &step_y, &lit_dummy(device)), + ); + } + let strength = post_uniform([0.9, 0.0, 0.0, 0.0]); + fullscreen( + &mut encoder, + &output, + &pipelines.composite, + &post_bind(shaded, &strength, &bloom_a), + ); + Some(gpu.read_back(encoder, &output, false)) +} + +/// A 1×1 texture for bind slots a pass doesn't read (a texture can't be both +/// the target and a binding of one pass). +fn lit_dummy(device: &wgpu::Device) -> wgpu::Texture { + device.create_texture(&wgpu::TextureDescriptor { + label: Some("scene unused"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: HDR, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) +} + +/// As [`lit_dummy`], for the shadow map slot while the shadow map is drawn. +fn lit_dummy_depth(device: &wgpu::Device) -> wgpu::Texture { + device.create_texture(&wgpu::TextureDescriptor { + label: Some("scene unused depth"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: DEPTH, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use zeron_visual::Token; + + fn palette(color: Color) -> [f32; 4] { + match color.token { + Token::Chart(0) => [0.3, 0.6, 0.95, 1.0], + Token::Chart(1) => [0.95, 0.45, 0.35, 1.0], + Token::Chart(2) => [0.4, 0.85, 0.5, 1.0], + Token::Chart(3) => [0.95, 0.8, 0.3, 1.0], + _ => [0.75, 0.75, 0.78, 1.0], + } + } + + fn room() -> Scene { + let mut visual = zeron_visual::Visual::load( + r#" + return function() + return ui.scene { height = 300, draw = function(s) + s:box { at = { 0, -0.05, 0 }, size = { 6, 0.1, 4 }, color = "muted" } + s:box { at = { -1.6, 0.5, 0 }, size = { 1, 1, 1 }, color = "chart1", roughness = 0.35 } + s:sphere { at = { 0, 0.6, 0.4 }, radius = 0.6, color = "chart2", metal = 1, roughness = 0.25 } + s:box { at = { 1.6, 0.6, 0 }, size = { 0.9, 1.2, 0.9 }, color = "chart3", glass = true, dispersion = 1 } + s:sphere { at = { 1.6, 1.6, -1 }, radius = 0.2, color = "chart4", glow = 4 } + end } + end + "#, + 1, + ) + .unwrap(); + let tree = visual.render().unwrap(); + let zeron_visual::NodeKind::Scene(node) = tree.kind else { + panic!("a scene") + }; + visual.draw_scene(node.draw, 0.0).unwrap() + } + + #[test] + fn a_room_renders_lit_with_glass_and_glow() { + if !crate::available() { + return; + } + let scene = room(); + let camera = Camera { + yaw: 0.6, + pitch: 0.45, + zoom: 1.0, + }; + // The first frame also makes the device and compiles pipelines. + render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + let start = std::time::Instant::now(); + let frame = render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + eprintln!( + "gpu scene {}x{} in {:?}", + frame.width, + frame.height, + start.elapsed() + ); + if let Ok(path) = std::env::var("ZERON_VISUAL_DUMP") { + crate::dump_ppm(&frame, &path); + } + assert_eq!((frame.width, frame.height), (1440, 720)); + let drawn = frame.bgra.chunks_exact(4).filter(|p| p[3] > 0).count(); + let total = (frame.width * frame.height) as usize; + assert!(drawn > total / 8, "geometry fills the frame: {drawn}"); + assert!(drawn < total, "and the background stays clear"); + } +} diff --git a/crates/visual-gpu/src/shader.rs b/crates/visual-gpu/src/shader.rs new file mode 100644 index 0000000000..505d14002f --- /dev/null +++ b/crates/visual-gpu/src/shader.rs @@ -0,0 +1,720 @@ +//! `ui.shader` on the GPU: buffer passes with feedback, the image pass, then +//! optional bloom and compositing by blend mode, read back as an image. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use wgpu::util::DeviceExt; +use zeron_visual::shader::{Blend, MAX_BUFFERS, ShaderUniforms}; + +use crate::{FORMAT, Frame, gpu}; + +/// Passes render in float, so feedback buffers keep precision and values +/// above 1 survive to the bloom. +const HDR: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; +const NOISE_SIZE: u32 = 256; + +/// What to run: the image pass, its buffers, and how it composites. +pub struct ShaderSpec<'a> { + pub image: &'a str, + pub buffers: &'a [String], + /// Named float inputs, in slot order. + pub inputs: &'a [String], + pub blend: Blend, + pub bloom: f32, +} + +/// One shader's textures between frames: each buffer's previous and +/// current frame, the image, and the post targets. Recreated when the size +/// or the number of buffers changes. +pub struct ShaderState { + size: (u32, u32), + buffers: Vec<[wgpu::Texture; 2]>, + /// Which of each pair holds the latest frame. + latest: usize, + image: wgpu::Texture, + bloom: [wgpu::Texture; 2], + output: wgpu::Texture, +} + +pub(crate) struct Pipelines { + layout: wgpu::BindGroupLayout, + pipeline_layout: wgpu::PipelineLayout, + fullscreen: wgpu::ShaderModule, + post_layout: wgpu::BindGroupLayout, + bright: wgpu::RenderPipeline, + blur: wgpu::RenderPipeline, + composite: wgpu::RenderPipeline, + clamp: wgpu::Sampler, + repeat: wgpu::Sampler, + noise: wgpu::Texture, + empty: wgpu::Texture, + passes: Mutex, String>>>, +} + +const FULLSCREEN: &str = "@vertex fn main(@builtin(vertex_index) i: u32) -> @builtin(position) vec4 { + let p = vec2(f32((i << 1u) & 2u), f32(i & 2u)); + return vec4(p * 2.0 - 1.0, 0.0, 1.0); +}"; + +const POST: &str = r#" +@group(0) @binding(0) var source: texture_2d; +@group(0) @binding(1) var source_sampler: sampler; +// Bright/blur: xy is one texel along the blur. Composite: x bloom strength, +// y blend mode (0 alpha, 1 light, 2 opaque). +@group(0) @binding(2) var post: vec4; +@group(0) @binding(3) var glow: texture_2d; + +struct Out { @builtin(position) clip: vec4, @location(0) uv: vec2 }; + +@vertex fn vs(@builtin(vertex_index) i: u32) -> Out { + let p = vec2(f32((i << 1u) & 2u), f32(i & 2u)); + var o: Out; + o.clip = vec4(p * 2.0 - 1.0, 0.0, 1.0); + o.uv = vec2(p.x, 1.0 - p.y); + return o; +} + +fn luma(c: vec3) -> f32 { return dot(c, vec3(0.2126, 0.7152, 0.0722)); } + +@fragment fn fs_bright(i: Out) -> @location(0) vec4 { + let c = textureSampleLevel(source, source_sampler, i.uv, 0.0).rgb; + // Soft knee from 0.6, so bright-but-not-clipped light still glows. + let l = luma(c); + let over = smoothstep(0.6, 1.2, l); + return vec4(c * over, 1.0); +} + +@fragment fn fs_blur(i: Out) -> @location(0) vec4 { + let w = array(0.227, 0.194, 0.121, 0.054, 0.016); + var sum = textureSampleLevel(source, source_sampler, i.uv, 0.0).rgb * w[0]; + for (var k = 1; k < 5; k++) { + let o = post.xy * f32(k) * 1.6; + sum += textureSampleLevel(source, source_sampler, i.uv + o, 0.0).rgb * w[k]; + sum += textureSampleLevel(source, source_sampler, i.uv - o, 0.0).rgb * w[k]; + } + return vec4(sum, 1.0); +} + +@fragment fn fs_composite(i: Out) -> @location(0) vec4 { + // Passes store rows bottom-up (Shadertoy's origin); turn them upright. + let uv = vec2(i.uv.x, 1.0 - i.uv.y); + let c = textureSampleLevel(source, source_sampler, uv, 0.0); + let g = textureSampleLevel(glow, source_sampler, uv, 0.0).rgb * post.x; + let rgb = clamp(c.rgb + g, vec3(0.0), vec3(1.0)); + if (post.y > 1.5) { + return vec4(rgb, 1.0); + } + if (post.y > 0.5) { + let a = max(rgb.r, max(rgb.g, rgb.b)); + if (a <= 0.0001) { return vec4(0.0); } + return vec4(rgb / a, a); + } + let a = clamp(c.a + luma(g), 0.0, 1.0); + return vec4(rgb, a); +} +"#; + +impl Pipelines { + fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Self { + let texture_entry = |binding| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + let sampler_entry = |binding| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }; + let uniform_entry = |binding| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }; + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("shader inputs"), + entries: &[ + uniform_entry(0), + texture_entry(1), + texture_entry(2), + texture_entry(3), + texture_entry(4), + texture_entry(5), + sampler_entry(6), + sampler_entry(7), + ], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("shader"), + bind_group_layouts: &[Some(&layout)], + immediate_size: 0, + }); + let fullscreen = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("fullscreen"), + source: wgpu::ShaderSource::Wgsl(FULLSCREEN.into()), + }); + let post_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("shader post"), + entries: &[ + texture_entry(0), + sampler_entry(1), + uniform_entry(2), + texture_entry(3), + ], + }); + let post_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("shader post"), + bind_group_layouts: &[Some(&post_layout)], + immediate_size: 0, + }); + let post = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("shader post"), + source: wgpu::ShaderSource::Wgsl(POST.into()), + }); + let post_pipeline = |label: &str, entry: &str, format| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&post_pipeline_layout), + vertex: wgpu::VertexState { + module: &post, + entry_point: Some("vs"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &post, + entry_point: Some(entry), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }; + let sampler = |address| { + device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("shader channel"), + address_mode_u: address, + address_mode_v: address, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }) + }; + // Independent random RGBA per texel, fixed so a shader looks the + // same on every device. + let mut state = 0x2545_f491_u32; + let noise_bytes: Vec = (0..NOISE_SIZE * NOISE_SIZE * 4) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 24) as u8 + }) + .collect(); + let noise = device.create_texture_with_data( + queue, + &wgpu::TextureDescriptor { + label: Some("shader noise"), + size: wgpu::Extent3d { + width: NOISE_SIZE, + height: NOISE_SIZE, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }, + wgpu::util::TextureDataOrder::LayerMajor, + &noise_bytes, + ); + let empty = device.create_texture_with_data( + queue, + &wgpu::TextureDescriptor { + label: Some("shader empty channel"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }, + wgpu::util::TextureDataOrder::LayerMajor, + &[0, 0, 0, 0], + ); + Self { + bright: post_pipeline("shader bloom bright", "fs_bright", HDR), + blur: post_pipeline("shader bloom blur", "fs_blur", HDR), + composite: post_pipeline("shader composite", "fs_composite", FORMAT), + layout, + pipeline_layout, + fullscreen, + post_layout, + clamp: sampler(wgpu::AddressMode::ClampToEdge), + repeat: sampler(wgpu::AddressMode::Repeat), + noise, + empty, + passes: Mutex::new(HashMap::new()), + } + } + + /// The pipeline for one complete fragment program, compiled once. + fn pass(&self, device: &wgpu::Device, program: &str) -> Result, String> { + let mut passes = self.passes.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(cached) = passes.get(program) { + return cached.clone(); + } + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("visual shader"), + source: wgpu::ShaderSource::Glsl { + shader: program.into(), + stage: wgpu::naga::ShaderStage::Fragment, + defines: &[], + }, + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("visual shader"), + layout: Some(&self.pipeline_layout), + vertex: wgpu::VertexState { + module: &self.fullscreen, + entry_point: Some("main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: HDR, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + let result = match pollster::block_on(scope.pop()) { + Some(error) => Err(format!("shader: {error}")), + None => Ok(Arc::new(pipeline)), + }; + // Keep the cache bounded; a session rarely holds more than a few. + if passes.len() > 64 { + passes.clear(); + } + passes.insert(program.to_owned(), result.clone()); + result + } +} + +fn texture(device: &wgpu::Device, label: &str, size: (u32, u32), format: wgpu::TextureFormat) -> wgpu::Texture { + device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: size.0, + height: size.1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }) +} + +impl ShaderState { + fn new(device: &wgpu::Device, size: (u32, u32), buffers: usize) -> Self { + let half = ((size.0 / 2).max(1), (size.1 / 2).max(1)); + Self { + size, + buffers: (0..buffers) + .map(|_| { + [ + texture(device, "shader buffer", size, HDR), + texture(device, "shader buffer", size, HDR), + ] + }) + .collect(), + latest: 0, + image: texture(device, "shader image", size, HDR), + bloom: [ + texture(device, "shader bloom", half, HDR), + texture(device, "shader bloom", half, HDR), + ], + output: texture(device, "shader output", size, FORMAT), + } + } +} + +/// Run one frame of a `ui.shader` at `width` by `height` device pixels. +/// `state` carries the buffers between frames; pass the same one each frame. +pub fn render_shader( + spec: &ShaderSpec, + uniforms: &ShaderUniforms, + width: u32, + height: u32, + state: &mut Option, +) -> Result { + let gpu = gpu().ok_or("this device has no GPU for shaders")?; + let width = width.clamp(1, 8192); + let height = height.clamp(1, 8192); + let buffers = spec.buffers.len().min(MAX_BUFFERS); + let device = &gpu.device; + let pipelines = gpu.shaders.get_or_init(|| Pipelines::new(device, &gpu.queue)); + // Compile every pass first, so an error leaves the state untouched. + let mut programs = Vec::with_capacity(buffers + 1); + for code in spec.buffers.iter().take(buffers).map(String::as_str).chain([spec.image]) { + let (program, _) = zeron_visual::shader::program(code, spec.inputs); + programs.push(pipelines.pass(device, &program)?); + } + let fresh = state + .as_ref() + .is_none_or(|s| s.size != (width, height) || s.buffers.len() != buffers); + if fresh { + *state = Some(ShaderState::new(device, (width, height), buffers)); + } + let state = state.as_mut().expect("just made"); + let (read, write) = (state.latest, 1 - state.latest); + + let mut uniforms = *uniforms; + for channel in &mut uniforms.channels { + *channel = [width as f32, height as f32, 1.0, 0.0]; + } + let uniform = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("shader inputs"), + contents: uniforms.as_bytes(), + usage: wgpu::BufferUsages::UNIFORM, + }); + let view = |t: &wgpu::Texture| t.create_view(&Default::default()); + let empty = view(&pipelines.empty); + let noise = view(&pipelines.noise); + // Channel `j` as pass `pass` sees it: an earlier buffer's frame from + // this frame, its own and later ones' from the last. + let channel = |pass: usize, j: usize| -> wgpu::TextureView { + if j >= buffers { + return empty.clone(); + } + let frame = if j < pass { write } else { read }; + view(&state.buffers[j][frame]) + }; + let mut encoder = device.create_command_encoder(&Default::default()); + for (pass, pipeline) in programs.iter().enumerate() { + let channels: Vec = (0..4).map(|j| channel(pass, j)).collect(); + let bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("shader inputs"), + layout: &pipelines.layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&channels[0]), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&channels[1]), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(&channels[2]), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: wgpu::BindingResource::TextureView(&channels[3]), + }, + wgpu::BindGroupEntry { + binding: 5, + resource: wgpu::BindingResource::TextureView(&noise), + }, + wgpu::BindGroupEntry { + binding: 6, + resource: wgpu::BindingResource::Sampler(&pipelines.clamp), + }, + wgpu::BindGroupEntry { + binding: 7, + resource: wgpu::BindingResource::Sampler(&pipelines.repeat), + }, + ], + }); + let target = if pass < buffers { + view(&state.buffers[pass][write]) + } else { + view(&state.image) + }; + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("shader pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &target, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + rp.set_pipeline(pipeline); + rp.set_bind_group(0, &bind, &[]); + rp.draw(0..3, 0..1); + } + + // Post: bloom, then composite by blend mode. + let post_uniform = |values: [f32; 4]| { + device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("shader post"), + contents: uniform_bytes(&values), + usage: wgpu::BufferUsages::UNIFORM, + }) + }; + let post_bind = |source: &wgpu::Texture, values: &wgpu::Buffer, glow: &wgpu::TextureView| { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("shader post"), + layout: &pipelines.post_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view(source)), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&pipelines.clamp), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: values.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(glow), + }, + ], + }) + }; + let fullscreen = |encoder: &mut wgpu::CommandEncoder, + target: &wgpu::Texture, + pipeline: &wgpu::RenderPipeline, + bind: &wgpu::BindGroup| { + let target = view(target); + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("shader post"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &target, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + rp.set_pipeline(pipeline); + rp.set_bind_group(0, bind, &[]); + rp.draw(0..3, 0..1); + }; + let glow = if spec.bloom > 0.0 { + let half = state.bloom[0].size(); + let zero = post_uniform([0.0; 4]); + let step_x = post_uniform([1.0 / half.width as f32, 0.0, 0.0, 0.0]); + let step_y = post_uniform([0.0, 1.0 / half.height as f32, 0.0, 0.0]); + fullscreen( + &mut encoder, + &state.bloom[0], + &pipelines.bright, + &post_bind(&state.image, &zero, &empty), + ); + for _ in 0..3 { + fullscreen( + &mut encoder, + &state.bloom[1], + &pipelines.blur, + &post_bind(&state.bloom[0], &step_x, &empty), + ); + fullscreen( + &mut encoder, + &state.bloom[0], + &pipelines.blur, + &post_bind(&state.bloom[1], &step_y, &empty), + ); + } + view(&state.bloom[0]) + } else { + empty.clone() + }; + let mode = match spec.blend { + Blend::Alpha => 0.0, + Blend::Light => 1.0, + Blend::Opaque => 2.0, + }; + let composite = post_uniform([spec.bloom, mode, 0.0, 0.0]); + fullscreen( + &mut encoder, + &state.output, + &pipelines.composite, + &post_bind(&state.image, &composite, &glow), + ); + let frame = gpu.read_back(encoder, &state.output, false); + state.latest = write; + Ok(frame) +} + +fn uniform_bytes(values: &[f32; 4]) -> &[u8] { + // SAFETY: four `f32`s. + unsafe { std::slice::from_raw_parts(values.as_ptr().cast::(), 16) } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec<'a>(image: &'a str, buffers: &'a [String], blend: Blend) -> ShaderSpec<'a> { + ShaderSpec { + image, + buffers, + inputs: &[], + blend, + bloom: 0.0, + } + } + + fn px(frame: &Frame, x: u32, y: u32) -> [u8; 4] { + let o = ((y * frame.width + x) * 4) as usize; + [frame.bgra[o + 2], frame.bgra[o + 1], frame.bgra[o], frame.bgra[o + 3]] + } + + #[test] + fn a_shader_renders_its_gradient_upright_with_time() { + if !crate::available() { + return; + } + let code = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + vec2 uv = fragCoord / iResolution.xy;\n\ + fragColor = vec4(uv.x, uv.y, fract(iTime), 1.0);\n\ + }"; + let mut uniforms = ShaderUniforms { + resolution: [64.0, 32.0, 1.0, 0.0], + ..Default::default() + }; + uniforms.time[0] = 0.5; + let frame = + render_shader(&spec(code, &[], Blend::Opaque), &uniforms, 64, 32, &mut None).unwrap(); + // Red grows to the right; green grows upward (Shadertoy's origin is + // bottom-left), so the top row is the greenest. + assert!(px(&frame, 60, 16)[0] > px(&frame, 4, 16)[0]); + assert!(px(&frame, 32, 2)[1] > px(&frame, 32, 29)[1]); + assert!((i32::from(px(&frame, 32, 16)[2]) - 128).abs() < 3, "iTime reaches the shader"); + } + + #[test] + fn light_blend_makes_black_clear_and_light_opaque() { + if !crate::available() { + return; + } + // A bright disc on black, as light-only art draws it. + let code = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + float d = length(fragCoord - iResolution.xy * 0.5);\n\ + fragColor = vec4(vec3(d < 8.0 ? 1.0 : 0.0), 1.0);\n\ + }"; + let uniforms = ShaderUniforms { + resolution: [48.0, 48.0, 1.0, 0.0], + ..Default::default() + }; + let frame = + render_shader(&spec(code, &[], Blend::Light), &uniforms, 48, 48, &mut None).unwrap(); + assert_eq!(px(&frame, 2, 2)[3], 0, "black is see-through"); + assert_eq!(px(&frame, 24, 24)[3], 255, "light is solid"); + } + + #[test] + fn a_buffer_reads_its_own_previous_frame() { + if !crate::available() { + return; + } + // Buffer A counts frames by adding 0.25 to what it wrote last time; + // the image shows it. + let buffer = vec!["void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + vec4 last = texture(iChannel0, fragCoord / iResolution.xy);\n\ + fragColor = vec4(last.r + 0.25, 0.0, 0.0, 1.0);\n\ + }" + .to_owned()]; + let image = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + fragColor = vec4(texture(iChannel0, fragCoord / iResolution.xy).r, 0.0, 0.0, 1.0);\n\ + }"; + let uniforms = ShaderUniforms { + resolution: [16.0, 16.0, 1.0, 0.0], + ..Default::default() + }; + let mut state = None; + let mut reds = Vec::new(); + for _ in 0..3 { + let frame = render_shader(&spec(image, &buffer, Blend::Opaque), &uniforms, 16, 16, &mut state) + .unwrap(); + reds.push(px(&frame, 8, 8)[0]); + } + assert!(reds[0] < reds[1] && reds[1] < reds[2], "accumulates: {reds:?}"); + assert!((i32::from(reds[2]) - 191).abs() < 4, "0.75 after three frames: {reds:?}"); + } + + #[test] + fn a_raymarched_mandelbulb_compiles_and_draws() { + if !crate::available() { + return; + } + let code = include_str!("../tests/mandelbulb.glsl"); + zeron_visual::shader::validate(code, &[]).unwrap(); + let uniforms = ShaderUniforms { + resolution: [96.0, 96.0, 1.0, 0.0], + camera: [0.6, 0.3, 1.0, 0.0], + ..Default::default() + }; + let frame = + render_shader(&spec(code, &[], Blend::Alpha), &uniforms, 96, 96, &mut None).unwrap(); + let lit = frame.bgra.chunks_exact(4).filter(|p| p[3] > 0).count(); + assert!(lit > 96 * 96 / 10, "the fractal covers the middle: {lit}"); + assert!(lit < 96 * 96, "and the sky stays clear"); + } +} diff --git a/crates/visual-gpu/tests/mandelbulb.glsl b/crates/visual-gpu/tests/mandelbulb.glsl new file mode 100644 index 0000000000..06b6f44d10 --- /dev/null +++ b/crates/visual-gpu/tests/mandelbulb.glsl @@ -0,0 +1,50 @@ +// A raymarched Mandelbulb in the style of cznull's volume shader: distance +// estimation, an orbit camera from iCamera, and a transparent sky. +float bulb(vec3 p, out float trap) { + vec3 z = p; + float dr = 1.0; + float r = 0.0; + trap = 1e9; + for (int i = 0; i < 8; i++) { + r = length(z); + if (r > 2.0) break; + trap = min(trap, r); + float theta = acos(clamp(z.z / r, -1.0, 1.0)) * 8.0; + float phi = atan(z.y, z.x) * 8.0; + dr = pow(r, 7.0) * 8.0 * dr + 1.0; + z = pow(r, 8.0) * vec3(sin(theta) * cos(phi), sin(phi) * sin(theta), cos(theta)) + p; + } + return 0.5 * log(r) * r / dr; +} + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y; + float yaw = iCamera.x; + float pitch = iCamera.y; + vec3 eye = vec3(cos(pitch) * sin(yaw), sin(pitch), cos(pitch) * cos(yaw)) * 2.6 / iCamera.z; + vec3 forward = normalize(-eye); + vec3 right = normalize(cross(forward, vec3(0.0, 1.0, 0.0))); + vec3 up = cross(right, forward); + vec3 dir = normalize(forward * 1.6 + uv.x * right + uv.y * up); + float t = 0.0; + float trap = 0.0; + bool hit = false; + for (int i = 0; i < 128; i++) { + float d = bulb(eye + dir * t, trap); + if (d < 0.0008 * t) { hit = true; break; } + t += d; + if (t > 6.0) break; + } + if (!hit) { fragColor = vec4(0.0); return; } + vec3 p = eye + dir * t; + vec2 e = vec2(0.0005, 0.0); + float tt; + vec3 n = normalize(vec3( + bulb(p + e.xyy, tt) - bulb(p - e.xyy, tt), + bulb(p + e.yxy, tt) - bulb(p - e.yxy, tt), + bulb(p + e.yyx, tt) - bulb(p - e.yyx, tt))); + vec3 light = normalize(vec3(0.3, 0.9, 0.3)); + float diffuse = max(dot(n, light), 0.0); + vec3 base = 0.5 + 0.5 * sin(trap * 10.0 + vec3(0.0, 2.05, -2.05)); + fragColor = vec4(base * (0.3 + 0.7 * diffuse), 1.0); +} diff --git a/crates/visual/Cargo.toml b/crates/visual/Cargo.toml index 5cc323039a..8d5bb82e81 100644 --- a/crates/visual/Cargo.toml +++ b/crates/visual/Cargo.toml @@ -8,4 +8,6 @@ description = "Sandboxed Luau runtime for agent visuals: scripts build a native [dependencies] mlua = { version = "0.12", features = ["luau"] } +# The GLSL compiler wgpu uses, so a shader is validated exactly as it will run. +naga = { version = "29", features = ["glsl-in"] } thiserror.workspace = true diff --git a/crates/visual/src/lib.rs b/crates/visual/src/lib.rs index 4d2e505fda..97b6c0c187 100644 --- a/crates/visual/src/lib.rs +++ b/crates/visual/src/lib.rs @@ -19,7 +19,9 @@ mod draw; mod node; +mod raster; mod scene; +pub mod shader; mod style; use std::cell::{Cell, RefCell}; @@ -31,9 +33,11 @@ use mlua::{Function, Lua, LuaOptions, StdLib, Table, Value, VmState}; pub use draw::{DrawCmd, DrawText, Pointer}; pub use node::{ Axis, Canvas, Chart, ChartKind, Column, HandlerId, Node, NodeKind, NumberFormat, SceneNode, + ShaderNode, Series, Span, Stack, Table_ as TableNode, Text, }; -pub use scene::{Camera, Prim, Projection, Scene}; +pub use raster::{Quality, Raster, RasterLabel}; +pub use scene::{Camera, CameraFrame, FaceView, Material, Prim, Projection, Scene, V3}; pub use style::{Align, CHART_SERIES, Color, Justify, TOKEN_NAMES, TextSize, Token, Weight}; /// Largest script `show_visual` accepts. @@ -428,6 +432,15 @@ fn exercise_draws( )); } } + NodeKind::Shader(shader) => { + let names: Vec = shader.inputs.iter().map(|(n, _)| n.clone()).collect(); + for (i, buffer) in shader.buffers.iter().enumerate() { + crate::shader::validate(buffer, &names).map_err(|e| { + VisualError::script(format!("in shader buffer {}: {e}", i + 1)) + })?; + } + crate::shader::validate(&shader.code, &names).map_err(VisualError::script)?; + } _ => {} } for child in children(node) { diff --git a/crates/visual/src/node.rs b/crates/visual/src/node.rs index 1270a982c0..d1be921fb5 100644 --- a/crates/visual/src/node.rs +++ b/crates/visual/src/node.rs @@ -56,6 +56,7 @@ pub enum NodeKind { Chart(Box), Canvas(Canvas), Scene(SceneNode), + Shader(ShaderNode), Button { label: String, on_click: HandlerId, @@ -239,6 +240,30 @@ pub struct SceneNode { pub zoom: f32, } +/// A fragment shader the desktop runs on the GPU (see [`crate::shader`]). +#[derive(Debug, Clone, PartialEq)] +pub struct ShaderNode { + pub height: f32, + /// GLSL defining `mainImage`: the image pass. + pub code: String, + /// Buffer passes, run in order before the image (`iChannel0`-`3`). + pub buffers: Vec, + /// Named number inputs, in slot order. + pub inputs: Vec<(String, f32)>, + /// Advance `iTime` every frame (on by default). + pub animate: bool, + /// Dragging orbits `iCamera` (otherwise it only moves `iMouse`). + pub orbit: bool, + pub spin: f32, + pub yaw: f32, + pub pitch: f32, + pub zoom: f32, + /// How the output sits on the chat. + pub blend: crate::shader::Blend, + /// Bloom strength on bright output (0 off). + pub bloom: f32, +} + pub const MIN_HEIGHT: f32 = 40.0; pub const MAX_HEIGHT: f32 = 720.0; const MAX_NODES: usize = 4000; @@ -453,6 +478,98 @@ impl Parser { zoom: (zoom as f32).clamp(0.2, 5.0), }) } + "shader" => { + props.allow(&[ + "height", + "code", + "inputs", + "animate", + "orbit", + "spin", + "camera", + "blend", + "bloom", + "buffers", + // Before `blend`: true kept the alpha, false was opaque. + "transparent", + ])?; + let code = match props.table.raw_get::("code")? { + Value::String(code) => code.to_string_lossy(), + Value::Nil => { + return Err(props.error( + "needs code: GLSL defining void mainImage(out vec4 fragColor, in \ + vec2 fragCoord)", + )); + } + _ => return Err(props.error("'code' must be a string of GLSL")), + }; + let mut buffers = Vec::new(); + if let Some(table) = props.get::>("buffers")? { + for value in table.sequence_values::() { + match value? { + Value::String(code) => buffers.push(code.to_string_lossy()), + _ => return Err(props.error("'buffers' is a list of GLSL strings")), + } + } + } + if buffers.len() > crate::shader::MAX_BUFFERS { + return Err(props.error(&format!( + "takes at most {} buffers", + crate::shader::MAX_BUFFERS + ))); + } + let blend = match (props.string("blend")?, props.bool("transparent")?) { + (Some(name), _) => crate::shader::Blend::parse(&name).ok_or_else(|| { + props.error("'blend' is \"alpha\", \"light\", or \"opaque\"") + })?, + (None, Some(true)) => crate::shader::Blend::Alpha, + (None, Some(false)) => crate::shader::Blend::Opaque, + (None, None) => crate::shader::Blend::default(), + }; + let mut inputs = Vec::new(); + if let Some(table) = props.get::>("inputs")? { + for pair in table.pairs::() { + let (name, value) = + pair.map_err(|_| props.error("'inputs' maps names to numbers"))?; + if !crate::shader::valid_input_name(&name) { + return Err(props.error(&format!( + "input '{name}' is not a usable GLSL name" + ))); + } + inputs.push((name, value as f32)); + } + } + if inputs.len() > crate::shader::MAX_INPUTS { + return Err(props.error(&format!( + "takes at most {} inputs", + crate::shader::MAX_INPUTS + ))); + } + // Slots follow the names, so a shader compiled for one set of + // values serves every other. + inputs.sort_by(|a, b| a.0.cmp(&b.0)); + let camera: Option
= props.get("camera")?; + let (mut yaw, mut pitch, mut zoom) = (0.0, 0.0, 1.0); + if let Some(camera) = camera { + yaw = camera.get::>("yaw")?.unwrap_or(yaw); + pitch = camera.get::>("pitch")?.unwrap_or(pitch); + zoom = camera.get::>("zoom")?.unwrap_or(zoom); + } + NodeKind::Shader(ShaderNode { + height: props.height()?, + code, + buffers, + inputs, + animate: props.bool("animate")?.unwrap_or(true), + orbit: props.bool("orbit")?.unwrap_or(false), + spin: props.number("spin")?.unwrap_or(0.0).clamp(-3.0, 3.0) as f32, + yaw: yaw as f32, + pitch: (pitch as f32).clamp(-1.5, 1.5), + zoom: (zoom as f32).clamp(0.2, 5.0), + blend, + bloom: props.number("bloom")?.unwrap_or(0.0).clamp(0.0, 3.0) as f32, + }) + } "button" => { props.allow(&["label", "on_click", "primary"])?; NodeKind::Button { diff --git a/crates/visual/src/prelude.luau b/crates/visual/src/prelude.luau index 6cf9f12fc6..f2b58e8e29 100644 --- a/crates/visual/src/prelude.luau +++ b/crates/visual/src/prelude.luau @@ -26,6 +26,7 @@ for _, kind in "chart", "canvas", "scene", + "shader", "button", "tabs", "toggle", diff --git a/crates/visual/src/raster.rs b/crates/visual/src/raster.rs new file mode 100644 index 0000000000..64e057fff7 --- /dev/null +++ b/crates/visual/src/raster.rs @@ -0,0 +1,772 @@ +//! Software rendering for `ui.scene`: a depth-buffered rasterizer that turns a +//! scene into one image the desktop paints. +//! +//! Sorting whole faces back to front (the painter's algorithm) cannot order a +//! wall against the furniture beside it, and painting every face as its own +//! vector path made orbiting slow. Here every sample keeps its depth, so +//! surfaces hide each other exactly; faces crossing the near plane are clipped +//! rather than dropped; curved solids shade smoothly; silhouettes and folds get +//! a fine outline; and edges are supersampled once the camera rests. The image +//! is straight-alpha BGRA, transparent wherever nothing was drawn, so a scene +//! still sits on glass and uses the theme's colors. + +use crate::scene::{Camera, Scene, V3, View, add, dot, normalize, scale, sub}; +use crate::style::{Color, TextSize}; + +/// How carefully to draw: `Draft` while the camera moves (one sample per +/// pixel), `Final` once it rests (supersampled within a budget). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Quality { + Draft, + Final, +} + +/// A rendered scene. +#[derive(Debug, Clone, Default)] +pub struct Raster { + /// Device pixels. + pub width: u32, + pub height: u32, + /// Straight-alpha BGRA, top row first. + pub bgra: Vec, + /// Hover hints: canvas position (logical pixels), radius, text. + pub hints: Vec<([f32; 2], f32, String)>, + /// Labels for the desktop to set as text over the image. + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RasterLabel { + pub at: [f32; 2], + pub text: String, + pub color: Color, + pub size: TextSize, +} + +/// Samples a `Final` render may use; past it, it draws one per pixel. +const SAMPLE_BUDGET: f32 = 6_000_000.0; +/// Where two neighbouring samples' depths differ by more than this fraction, +/// a fold or silhouette is outlined. +const EDGE_DEPTH: f32 = 0.02; +/// How much an outline darkens the surface it crosses. +const EDGE_SHADE: f32 = 0.78; + +#[derive(Clone, Copy)] +struct Vertex { + /// View space. + p: V3, + /// Lit color, straight sRGB with alpha. + c: [f32; 4], +} + +/// The sample grid: `k` samples per logical pixel. +struct Grid { + width: usize, + height: usize, + k: f32, +} + +/// Rows `y0..y1` of the sample grid, rendered by one thread. +struct Band<'a> { + y0: usize, + y1: usize, + width: usize, + /// Premultiplied sRGB. + color: &'a mut [[f32; 4]], + /// View-space depth of the nearest surface; infinity where empty. + depth: &'a mut [f32], +} + +/// A dot in sample space. +struct Dot { + x: f32, + y: f32, + z: f32, + r: f32, + c: [f32; 4], +} + +impl Scene { + /// Draw the scene for a canvas `width` by `height` logical pixels at + /// `pixel_scale` device pixels each. `rgba` resolves a scene color to + /// straight sRGB components in 0..1. Rows render in parallel. + pub fn render( + &self, + camera: Camera, + width: f32, + height: f32, + pixel_scale: f32, + quality: Quality, + rgba: &dyn Fn(Color) -> [f32; 4], + ) -> Raster { + let device_w = (width * pixel_scale).ceil().max(0.0) as usize; + let device_h = (height * pixel_scale).ceil().max(0.0) as usize; + if device_w == 0 || device_h == 0 { + return Raster::default(); + } + let Some(view) = self.view_for(camera, width, height) else { + return Raster { + width: device_w as u32, + height: device_h as u32, + bgra: vec![0; device_w * device_h * 4], + ..Default::default() + }; + }; + let ss = match quality { + Quality::Draft => 1, + Quality::Final if (device_w * device_h * 4) as f32 <= SAMPLE_BUDGET => 2, + Quality::Final => 1, + }; + let grid = Grid { + width: device_w * ss, + height: device_h * ss, + k: pixel_scale * ss as f32, + }; + + // Everything in sample space first (cheap, single-threaded); the + // per-sample work after splits across cores. + let mut opaque: Vec<[Sample; 3]> = Vec::new(); + let mut translucent: Vec<(f32, Vec<[Sample; 3]>)> = Vec::new(); + for face in &self.faces { + let Some(polygon) = shade_face(face, &view, rgba) else { + continue; + }; + let samples: Vec = polygon.iter().map(|v| to_sample(&grid, &view, v)).collect(); + let tris = (1..samples.len() - 1).map(|i| [samples[0], samples[i], samples[i + 1]]); + if polygon.iter().any(|v| v.c[3] < 0.999) { + let depth = polygon.iter().map(|v| v.p[2]).sum::() / polygon.len() as f32; + translucent.push((depth, tris.collect())); + } else { + opaque.extend(tris); + } + } + // See-through surfaces blend far to near over what they cover. + translucent.sort_by(|a, b| b.0.total_cmp(&a.0)); + let translucent: Vec<[Sample; 3]> = translucent.into_iter().flat_map(|(_, t)| t).collect(); + // Lines and points ride just in front of the surface they lie on. + let bias = view.radius * 2e-3; + let mut lines: Vec<[Sample; 3]> = Vec::new(); + for line in &self.lines { + line_tris(&grid, &view, line.a, line.b, line.width, rgba(line.color), &mut lines); + } + let (hints, labels) = self.annotations(camera, width, height); + let mut dots = Vec::new(); + for point in &self.points { + let p = view.view(point.at); + if p[2] < view.near { + continue; + } + let at = view.screen(p); + dots.push(Dot { + x: at[0] * grid.k, + y: at[1] * grid.k, + z: p[2] - bias, + r: point.radius * grid.k, + c: rgba(point.color), + }); + } + let mut color = vec![[0.0f32; 4]; grid.width * grid.height]; + let mut depth = vec![f32::INFINITY; grid.width * grid.height]; + // Opaque surfaces, writing depth. + for_bands(&grid, ss, &mut color, &mut depth, |band| { + for tri in &opaque { + fill_triangle(band, *tri, true, 0.0); + } + }); + outline(&grid, ss, &mut color, &depth); + // Then everything that blends without hiding what is behind it. + for_bands(&grid, ss, &mut color, &mut depth, |band| { + for tri in &translucent { + fill_triangle(band, *tri, false, 0.0); + } + for tri in &lines { + fill_triangle(band, *tri, false, bias); + } + for dot in &dots { + draw_dot(band, dot); + } + }); + + Raster { + width: device_w as u32, + height: device_h as u32, + bgra: resolve(&grid, &color, device_w, device_h, ss), + hints, + labels, + } + } +} + +impl Scene { + /// Hover hints and labels at their canvas positions, for the desktop to + /// draw as text over whichever renderer made the image. + #[allow(clippy::type_complexity)] + pub fn annotations( + &self, + camera: Camera, + width: f32, + height: f32, + ) -> (Vec<([f32; 2], f32, String)>, Vec) { + let Some(view) = self.view_for(camera, width, height) else { + return Default::default(); + }; + let hints = self + .points + .iter() + .filter_map(|point| { + let p = view.view(point.at); + let hint = point.hint.as_ref()?; + (p[2] >= view.near).then(|| (view.screen(p), point.radius.max(4.0), hint.clone())) + }) + .collect(); + let labels = self + .labels + .iter() + .filter_map(|label| { + let p = view.view(label.at); + (p[2] >= view.near).then(|| RasterLabel { + at: view.screen(p), + text: label.text.clone(), + color: label.color, + size: label.size, + }) + }) + .collect(); + (hints, labels) + } +} + +/// How many threads share a render. +fn threads() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .clamp(1, 16) +} + +/// Run `work` on horizontal bands of the grid, one per thread. Bands start on +/// whole output rows (`ss` sample rows), so supersampled pixels never straddle +/// two bands. +fn for_bands( + grid: &Grid, + ss: usize, + color: &mut [[f32; 4]], + depth: &mut [f32], + work: impl Fn(&mut Band) + Sync, +) { + let rows = grid.height / ss; + let per = rows.div_ceil(threads()).max(1) * ss; + let width = grid.width; + std::thread::scope(|scope| { + let mut color = color; + let mut depth = depth; + let mut y0 = 0; + let work = &work; + while y0 < grid.height { + let y1 = (y0 + per).min(grid.height); + let (band_color, rest_color) = color.split_at_mut((y1 - y0) * width); + let (band_depth, rest_depth) = depth.split_at_mut((y1 - y0) * width); + color = rest_color; + depth = rest_depth; + let mut band = Band { + y0, + y1, + width, + color: band_color, + depth: band_depth, + }; + scope.spawn(move || work(&mut band)); + y0 = y1; + } + }); +} + +/// A face as lit view-space vertices, clipped to the near plane; `None` when +/// it is behind the camera or turned away. +fn shade_face( + face: &crate::scene::Face, + view: &View, + rgba: &dyn Fn(Color) -> [f32; 4], +) -> Option> { + let corners = &face.v[..face.n as usize]; + let centroid = scale( + corners.iter().fold([0.0; 3], |acc, v| add(acc, *v)), + 1.0 / corners.len() as f32, + ); + let toward_eye = dot(face.normal, sub(view.eye, centroid)); + if !face.two_sided && toward_eye < 0.0 { + return None; + } + // The lit side of a two-sided surface is the one the camera sees. + let flip = if toward_eye < 0.0 { -1.0 } else { 1.0 }; + let mut base = rgba(face.color); + // Without the GPU's refraction, glass is a see-through tint. + base[3] *= 1.0 - 0.7 * face.material.glass; + let glow = face.material.glow; + let polygon: Vec = corners + .iter() + .enumerate() + .map(|(i, corner)| { + let normal = match &face.normals { + Some(normals) => normals[i], + None => face.normal, + }; + Vertex { + p: view.view(*corner), + c: lit(base, scale(normal, flip), view.light, glow), + } + }) + .collect(); + let clipped = clip_near(&polygon, view.near); + (clipped.len() >= 3).then_some(clipped) +} + +/// A soft studio light: sky-to-ground ambient, a key light from the upper +/// left, and a dim fill from the opposite side. Lit in linear light. +fn lit(base: [f32; 4], normal: V3, key: V3, glow: f32) -> [f32; 4] { + let n = normalize(normal); + let sky = 0.5 + 0.5 * n[1]; + let ambient = 0.34 + 0.2 * sky; + let key_light = dot(n, key).max(0.0) * 0.6; + let fill = dot(n, scale(key, -1.0)).max(0.0) * 0.14; + let light = ambient + key_light + fill + glow; + let mut out = base; + for channel in &mut out[..3] { + *channel = to_srgb((to_linear(*channel) * light).min(1.0)); + } + out +} + +fn to_linear(c: f32) -> f32 { + if c <= 0.04045 { + c / 12.92 + } else { + ((c + 0.055) / 1.055).powf(2.4) + } +} + +fn to_srgb(c: f32) -> f32 { + if c <= 0.0031308 { + c * 12.92 + } else { + 1.055 * c.powf(1.0 / 2.4) - 0.055 + } +} + +/// Sutherland–Hodgman against `z = near`: keep what is in front. +fn clip_near(polygon: &[Vertex], near: f32) -> Vec { + if polygon.iter().all(|v| v.p[2] >= near) { + return polygon.to_vec(); + } + let mut out = Vec::with_capacity(polygon.len() + 2); + for i in 0..polygon.len() { + let a = polygon[i]; + let b = polygon[(i + 1) % polygon.len()]; + let (a_in, b_in) = (a.p[2] >= near, b.p[2] >= near); + if a_in { + out.push(a); + } + if a_in != b_in { + let t = (near - a.p[2]) / (b.p[2] - a.p[2]); + out.push(Vertex { + p: add(a.p, scale(sub(b.p, a.p), t)), + c: lerp4(a.c, b.c, t), + }); + } + } + out +} + +fn lerp4(a: [f32; 4], b: [f32; 4], t: f32) -> [f32; 4] { + [ + a[0] + (b[0] - a[0]) * t, + a[1] + (b[1] - a[1]) * t, + a[2] + (b[2] - a[2]) * t, + a[3] + (b[3] - a[3]) * t, + ] +} + +/// A vertex in sample space: x, y, view depth, color. +#[derive(Clone, Copy)] +struct Sample { + x: f32, + y: f32, + z: f32, + c: [f32; 4], +} + +fn to_sample(grid: &Grid, view: &View, v: &Vertex) -> Sample { + let [x, y] = view.screen(v.p); + Sample { + x: x * grid.k, + y: y * grid.k, + z: v.p[2], + c: v.c, + } +} + +/// Rasterize one triangle into a band with perspective-correct depth and +/// color. `opaque` writes depth and replaces color; otherwise it blends. +fn fill_triangle(band: &mut Band, tri: [Sample; 3], opaque: bool, bias: f32) { + let [a, b, c] = tri; + let area = (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); + if area.abs() < 1e-8 { + return; + } + let min_y = (a.y.min(b.y).min(c.y).floor().max(band.y0 as f32)) as usize; + let max_y = (a.y.max(b.y).max(c.y).ceil() as isize).min(band.y1 as isize - 1); + if max_y < min_y as isize { + return; + } + let min_x = a.x.min(b.x).min(c.x).floor().max(0.0) as usize; + let max_x = (a.x.max(b.x).max(c.x).ceil() as isize).min(band.width as isize - 1); + if max_x < min_x as isize { + return; + } + let inv_area = 1.0 / area; + let inv_z = [1.0 / a.z, 1.0 / b.z, 1.0 / c.z]; + for y in min_y..=max_y as usize { + let py = y as f32 + 0.5; + let row = (y - band.y0) * band.width; + for x in min_x..=max_x as usize { + let px = x as f32 + 0.5; + let w0 = ((b.x - px) * (c.y - py) - (b.y - py) * (c.x - px)) * inv_area; + let w1 = ((c.x - px) * (a.y - py) - (c.y - py) * (a.x - px)) * inv_area; + let w2 = 1.0 - w0 - w1; + if w0 < 0.0 || w1 < 0.0 || w2 < 0.0 { + continue; + } + let (p0, p1, p2) = (w0 * inv_z[0], w1 * inv_z[1], w2 * inv_z[2]); + let one_over_z = p0 + p1 + p2; + let z = 1.0 / one_over_z - bias; + let i = row + x; + if z >= band.depth[i] { + continue; + } + let color = [ + (p0 * a.c[0] + p1 * b.c[0] + p2 * c.c[0]) / one_over_z, + (p0 * a.c[1] + p1 * b.c[1] + p2 * c.c[1]) / one_over_z, + (p0 * a.c[2] + p1 * b.c[2] + p2 * c.c[2]) / one_over_z, + (p0 * a.c[3] + p1 * b.c[3] + p2 * c.c[3]) / one_over_z, + ]; + if opaque { + band.depth[i] = z; + band.color[i] = [color[0], color[1], color[2], 1.0]; + } else { + blend(&mut band.color[i], color); + } + } + } +} + +/// Straight color over a premultiplied sample. +fn blend(under: &mut [f32; 4], over: [f32; 4]) { + let a = over[3].clamp(0.0, 1.0); + for ch in 0..3 { + under[ch] = over[ch] * a + under[ch] * (1.0 - a); + } + under[3] = a + under[3] * (1.0 - a); +} + +/// Darken samples on a silhouette or fold, where depth jumps between +/// neighbours, so shapes read clearly whatever the lighting. +fn outline(grid: &Grid, ss: usize, color: &mut [[f32; 4]], depth: &[f32]) { + let (w, h) = (grid.width, grid.height); + let rows = h / ss; + let per = rows.div_ceil(threads()).max(1) * ss; + std::thread::scope(|scope| { + let mut rest = color; + let mut y0 = 0; + while y0 < h { + let y1 = (y0 + per).min(h); + let (band, tail) = rest.split_at_mut((y1 - y0) * w); + rest = tail; + scope.spawn(move || { + for y in y0..y1 { + for x in 0..w { + let i = y * w + x; + let z = depth[i]; + if !z.is_finite() { + continue; + } + let jump = |j: usize| { + let other = depth[j]; + !other.is_finite() || (other - z).abs() > EDGE_DEPTH * z.min(other) + }; + let edge = (x + 1 < w && jump(i + 1)) + || (x > 0 && jump(i - 1)) + || (y + 1 < h && jump(i + w)) + || (y > 0 && jump(i - w)); + if edge { + for ch in &mut band[(y - y0) * w + x][..3] { + *ch *= EDGE_SHADE; + } + } + } + } + }); + y0 = y1; + } + }); +} + +/// A line `width` logical pixels wide, as two triangles in sample space. +#[allow(clippy::too_many_arguments)] +fn line_tris( + grid: &Grid, + view: &View, + a: V3, + b: V3, + width: f32, + color: [f32; 4], + out: &mut Vec<[Sample; 3]>, +) { + let (mut a, mut b) = (view.view(a), view.view(b)); + if a[2] < view.near && b[2] < view.near { + return; + } + // Clip the part behind the near plane. + if a[2] < view.near || b[2] < view.near { + let t = (view.near - a[2]) / (b[2] - a[2]); + let cut = add(a, scale(sub(b, a), t)); + if a[2] < view.near { + a = cut; + } else { + b = cut; + } + } + let to = |p: V3| { + let [x, y] = view.screen(p); + (x * grid.k, y * grid.k, p[2]) + }; + let (ax, ay, az) = to(a); + let (bx, by, bz) = to(b); + let (dx, dy) = (bx - ax, by - ay); + let len = dx.hypot(dy); + if len < 1e-4 { + return; + } + let half = (width * grid.k).max(1.0) / 2.0; + let (nx, ny) = (-dy / len * half, dx / len * half); + let corner = |x: f32, y: f32, z: f32| Sample { x, y, z, c: color }; + let quad = [ + corner(ax + nx, ay + ny, az), + corner(bx + nx, by + ny, bz), + corner(bx - nx, by - ny, bz), + corner(ax - nx, ay - ny, az), + ]; + out.push([quad[0], quad[1], quad[2]]); + out.push([quad[0], quad[2], quad[3]]); +} + +/// A round dot, depth-tested at its center. +fn draw_dot(band: &mut Band, dot: &Dot) { + let min_y = (dot.y - dot.r).floor().max(band.y0 as f32) as usize; + let max_y = ((dot.y + dot.r).ceil() as isize).min(band.y1 as isize - 1); + let min_x = (dot.x - dot.r).floor().max(0.0) as usize; + let max_x = ((dot.x + dot.r).ceil() as isize).min(band.width as isize - 1); + if max_y < min_y as isize || max_x < min_x as isize { + return; + } + for y in min_y..=max_y as usize { + for x in min_x..=max_x as usize { + let (px, py) = (x as f32 + 0.5 - dot.x, y as f32 + 0.5 - dot.y); + if px * px + py * py > dot.r * dot.r { + continue; + } + let i = (y - band.y0) * band.width + x; + if dot.z < band.depth[i] { + blend(&mut band.color[i], dot.c); + } + } + } +} + +/// Average each pixel's samples and hand back straight-alpha BGRA. +fn resolve(grid: &Grid, color: &[[f32; 4]], width: usize, height: usize, ss: usize) -> Vec { + let mut out = vec![0u8; width * height * 4]; + let n = (ss * ss) as f32; + let per = height.div_ceil(threads()).max(1); + std::thread::scope(|scope| { + for (chunk, rows) in out.chunks_mut(per * width * 4).enumerate() { + scope.spawn(move || { + let y0 = chunk * per; + for (dy, row) in rows.chunks_mut(width * 4).enumerate() { + let y = y0 + dy; + for x in 0..width { + let mut sum = [0.0f32; 4]; + for sy in 0..ss { + let base = (y * ss + sy) * grid.width + x * ss; + for sample in &color[base..base + ss] { + for ch in 0..4 { + sum[ch] += sample[ch]; + } + } + } + let alpha = sum[3] / n; + if alpha <= 0.0 { + continue; + } + let straight = + |ch: usize| ((sum[ch] / n / alpha).clamp(0.0, 1.0) * 255.0).round() as u8; + let o = x * 4; + row[o] = straight(2); + row[o + 1] = straight(1); + row[o + 2] = straight(0); + row[o + 3] = (alpha.clamp(0.0, 1.0) * 255.0).round() as u8; + } + } + }); + } + }); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::style::Token; + + fn camera() -> Camera { + Camera { + yaw: 0.0, + pitch: 0.0, + zoom: 1.0, + } + } + + fn colors(color: Color) -> [f32; 4] { + match color.token { + Token::Chart(0) => [1.0, 0.0, 0.0, 1.0], + Token::Chart(1) => [0.0, 0.0, 1.0, 1.0], + _ => [0.5, 0.5, 0.5, 1.0], + } + } + + fn pixel(raster: &Raster, x: u32, y: u32) -> [u8; 4] { + let o = ((y * raster.width + x) * 4) as usize; + let p = &raster.bgra[o..o + 4]; + [p[2], p[1], p[0], p[3]] + } + + /// A small red square in front of a large blue one: whatever order the + /// faces come in, the near one covers the far one where they overlap. + #[test] + fn nearer_surfaces_hide_farther_ones_whatever_their_order() { + for near_first in [true, false] { + let mut s = Scene::default(); + let far = [[-2., -2., -1.], [2., -2., -1.], [2., 2., -1.], [-2., 2., -1.]]; + let near = [[-0.5, -0.5, 1.], [0.5, -0.5, 1.], [0.5, 0.5, 1.], [-0.5, 0.5, 1.]]; + let faces = [(near, 0), (far, 1)]; + let order: Vec<_> = if near_first { + faces.to_vec() + } else { + faces.iter().rev().copied().collect() + }; + for (quad, chart) in order { + s.face(&quad, Color::token(Token::Chart(chart)), Some([0., 0., 1.]), true) + .unwrap(); + } + let raster = s.render(camera(), 200.0, 200.0, 1.0, Quality::Draft, &colors); + let center = pixel(&raster, 100, 100); + assert!(center[0] > 100 && center[2] < 60, "near red wins: {center:?}"); + let corner = pixel(&raster, 100, 30); + assert!(corner[2] > 100 && corner[0] < 60, "far blue shows around it: {corner:?}"); + // Nothing outside the geometry: transparent. + assert_eq!(pixel(&raster, 1, 1)[3], 0); + } + } + + #[test] + fn faces_crossing_the_camera_are_clipped_not_dropped() { + let mut s = Scene::default(); + // A long floor that runs from far away to behind the camera. + let floor = [[-1., -1., -40.], [1., -1., -40.], [1., -1., 40.], [-1., -1., 40.]]; + s.face(&floor, Color::token(Token::Chart(0)), Some([0., 1., 0.]), true) + .unwrap(); + // A marker the camera frames itself on, near the middle. + s.face( + &[[-0.2, 0., 0.], [0.2, 0., 0.], [0., 0.3, 0.]], + Color::token(Token::Chart(1)), + Some([0., 0., 1.]), + true, + ) + .unwrap(); + let zoomed = Camera { + zoom: 6.0, + ..camera() + }; + let raster = s.render(zoomed, 200.0, 200.0, 1.0, Quality::Draft, &colors); + let drawn = raster.bgra.chunks_exact(4).filter(|p| p[3] > 0).count(); + assert!(drawn > 0, "the floor still shows"); + } + + #[test] + fn final_quality_supersamples_within_its_budget() { + let mut s = Scene::default(); + s.face( + &[[-1., -1., 0.], [1., -1., 0.], [0., 1., 0.]], + Color::token(Token::Chart(0)), + Some([0., 0., 1.]), + true, + ) + .unwrap(); + let draft = s.render(camera(), 100.0, 80.0, 2.0, Quality::Draft, &colors); + let fine = s.render(camera(), 100.0, 80.0, 2.0, Quality::Final, &colors); + assert_eq!((draft.width, draft.height), (200, 160)); + assert_eq!((fine.width, fine.height), (200, 160)); + // Supersampled edges have partial coverage; one sample cannot. + let partial = |r: &Raster| r.bgra.chunks_exact(4).any(|p| p[3] > 0 && p[3] < 255); + assert!(!partial(&draft)); + assert!(partial(&fine)); + } +} + +#[cfg(test)] +mod timing { + use super::*; + use crate::style::Token; + + /// Not an assertion: prints how long a furnished room takes, so a change + /// that slows rendering shows up in `cargo test -- --nocapture timing`. + #[test] + fn timing_a_furnished_room() { + let mut s = Scene::default(); + let quad = |s: &mut Scene, q: [V3; 4], c: u8| { + s.face(&q, Color::token(Token::Chart(c)), None, true).unwrap(); + }; + quad(&mut s, [[-4., 0., -3.], [4., 0., -3.], [4., 0., 3.], [-4., 0., 3.]], 0); + quad(&mut s, [[-4., 0., -3.], [4., 0., -3.], [4., 3., -3.], [-4., 3., -3.]], 1); + quad(&mut s, [[-4., 0., -3.], [-4., 0., 3.], [-4., 3., 3.], [-4., 3., -3.]], 2); + // Spheres stand in for plants and lamps: smooth, many faces. + for k in 0..8 { + let at = [-3.0 + k as f32 * 0.8, 0.6, 1.0]; + let (lon, lat) = (36, 18); + for j in 0..lat { + for i in 0..lon { + let p = |i: u32, j: u32| { + let t = std::f32::consts::TAU * i as f32 / lon as f32; + let f = std::f32::consts::PI * j as f32 / lat as f32; + [at[0] + 0.4 * f.sin() * t.cos(), at[1] + 0.4 * f.cos(), at[2] + 0.4 * f.sin() * t.sin()] + }; + let q = [p(i, j), p(i + 1, j), p(i + 1, j + 1), p(i, j + 1)]; + let n: Vec = q.iter().map(|c| sub(*c, at)).collect(); + s.smooth_face(&q, &n, Color::token(Token::Chart(3))).unwrap(); + } + } + } + let colors = |_: Color| [0.6, 0.6, 0.6, 1.0]; + let camera = Camera { yaw: 0.7, pitch: 0.4, zoom: 1.0 }; + for quality in [Quality::Draft, Quality::Final] { + let start = std::time::Instant::now(); + let raster = s.render(camera, 736.0, 420.0, 2.0, quality, &colors); + println!( + "{quality:?}: {} faces, {}x{} in {:?}", + s.face_count(), + raster.width, + raster.height, + start.elapsed() + ); + } + } +} diff --git a/crates/visual/src/scene.rs b/crates/visual/src/scene.rs index 01e6f5b95a..7b64f7eb63 100644 --- a/crates/visual/src/scene.rs +++ b/crates/visual/src/scene.rs @@ -17,49 +17,144 @@ const MAX_ITEMS: usize = 20_000; pub type V3 = [f32; 3]; #[derive(Debug, Clone, PartialEq)] -struct Face { - v: [V3; 4], +pub(crate) struct Face { + pub(crate) v: [V3; 4], /// 3 or 4 corners. - n: u8, - normal: V3, - color: Color, + pub(crate) n: u8, + pub(crate) normal: V3, + /// Per-corner normals of a curved solid (spheres, cylinder sides), so it + /// shades smoothly instead of faceted. + pub(crate) normals: Option<[V3; 4]>, + pub(crate) color: Color, /// Closed solids cull faces turned away from the camera; open surfaces /// and meshes light both sides. - two_sided: bool, + pub(crate) two_sided: bool, + pub(crate) material: Material, +} + +/// How a surface takes light. `glass` lets light through (refracting by +/// `ior`, splitting into colors by `dispersion`); `glow` emits light of the +/// surface's own color, which blooms. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Material { + /// 0 is a mirror finish, 1 fully matte. + pub roughness: f32, + /// 0 is a dielectric (plastic, wood), 1 a metal. + pub metalness: f32, + pub glow: f32, + /// 0 opaque to 1 clear glass. + pub glass: f32, + pub ior: f32, + pub dispersion: f32, +} + +impl Default for Material { + fn default() -> Self { + Self { + roughness: 0.55, + metalness: 0.0, + glow: 0.0, + glass: 0.0, + ior: 1.5, + dispersion: 0.0, + } + } +} + +/// A face as the GPU renderer reads it. +pub struct FaceView<'a> { + pub corners: &'a [V3], + pub normal: V3, + /// Per-corner normals of a curved solid. + pub normals: Option<&'a [V3]>, + pub color: Color, + pub two_sided: bool, + pub material: Material, +} + +/// The framed orbit camera, for a renderer that does its own projection. +#[derive(Debug, Clone, Copy)] +pub struct CameraFrame { + pub eye: V3, + pub right: V3, + pub up: V3, + /// Into the screen. + pub forward: V3, + /// Pixels per unit at unit depth, for the canvas the frame was made for. + pub focal: f32, + pub near: f32, + /// The geometry's center and bounding radius. + pub center: V3, + pub radius: f32, } #[derive(Debug, Clone, PartialEq)] -struct Line3 { - a: V3, - b: V3, - color: Color, - width: f32, +pub(crate) struct Line3 { + pub(crate) a: V3, + pub(crate) b: V3, + pub(crate) color: Color, + pub(crate) width: f32, } #[derive(Debug, Clone, PartialEq)] -struct Point3 { - at: V3, - radius: f32, - color: Color, - hint: Option, +pub(crate) struct Point3 { + pub(crate) at: V3, + pub(crate) radius: f32, + pub(crate) color: Color, + pub(crate) hint: Option, } #[derive(Debug, Clone, PartialEq)] -struct Label3 { - at: V3, - text: String, - color: Color, - size: TextSize, +pub(crate) struct Label3 { + pub(crate) at: V3, + pub(crate) text: String, + pub(crate) color: Color, + pub(crate) size: TextSize, } /// The geometry one draw call produced. #[derive(Debug, Default, Clone, PartialEq)] pub struct Scene { - faces: Vec, - lines: Vec, - points: Vec, - labels: Vec, + pub(crate) faces: Vec, + pub(crate) lines: Vec, + pub(crate) points: Vec, + pub(crate) labels: Vec, pub time: f64, + /// The material the primitive being added gives its faces. + pub(crate) material: Material, +} + +/// The camera a scene is seen through: framed on its geometry and orbited. +pub(crate) struct View { + pub(crate) eye: V3, + right: V3, + up: V3, + forward: V3, + focal: f32, + pub(crate) near: f32, + cx: f32, + cy: f32, + pub(crate) center: V3, + /// The geometry's bounding radius. + pub(crate) radius: f32, + /// The soft key light the painter shades with, from the upper left. + pub(crate) light: V3, +} + +impl View { + /// World to view space: x right, y up, z into the screen. + pub(crate) fn view(&self, p: V3) -> V3 { + let d = sub(p, self.eye); + [dot(d, self.right), dot(d, self.up), dot(d, self.forward)] + } + + /// View space to canvas pixels. + pub(crate) fn screen(&self, v: V3) -> [f32; 2] { + [ + self.cx + self.focal * v[0] / v[2], + self.cy - self.focal * v[1] / v[2], + ] + } } /// The orbit camera: angles in radians around the geometry's center. @@ -106,16 +201,16 @@ pub struct Projection { pub hints: Vec<([f32; 2], f32, String)>, } -fn sub(a: V3, b: V3) -> V3 { +pub(crate) fn sub(a: V3, b: V3) -> V3 { [a[0] - b[0], a[1] - b[1], a[2] - b[2]] } -fn add(a: V3, b: V3) -> V3 { +pub(crate) fn add(a: V3, b: V3) -> V3 { [a[0] + b[0], a[1] + b[1], a[2] + b[2]] } -fn scale(a: V3, s: f32) -> V3 { +pub(crate) fn scale(a: V3, s: f32) -> V3 { [a[0] * s, a[1] * s, a[2] * s] } -fn dot(a: V3, b: V3) -> f32 { +pub(crate) fn dot(a: V3, b: V3) -> f32 { a[0] * b[0] + a[1] * b[1] + a[2] * b[2] } fn cross(a: V3, b: V3) -> V3 { @@ -125,7 +220,7 @@ fn cross(a: V3, b: V3) -> V3 { a[0] * b[1] - a[1] * b[0], ] } -fn normalize(a: V3) -> V3 { +pub(crate) fn normalize(a: V3) -> V3 { let len = dot(a, a).sqrt(); if len < 1e-9 { [0.0, 1.0, 0.0] @@ -142,7 +237,7 @@ impl Scene { } } - fn face( + pub(crate) fn face( &mut self, corners: &[V3], color: Color, @@ -179,12 +274,28 @@ impl Scene { v, n: corners.len() as u8, normal, + normals: None, color, two_sided, + material: self.material, }); Ok(()) } + /// A face of a curved solid, shaded from the surface's own normals. + pub(crate) fn smooth_face(&mut self, corners: &[V3], normals: &[V3], color: Color) -> mlua::Result<()> { + let outward = normals.iter().fold([0.0; 3], |acc, n| add(acc, *n)); + self.face(corners, color, Some(outward), false)?; + let mut n = [[0.0; 3]; 4]; + for (slot, normal) in n.iter_mut().zip(normals) { + *slot = normalize(*normal); + } + if let Some(face) = self.faces.last_mut() { + face.normals = Some(n); + } + Ok(()) + } + fn polygon( &mut self, corners: &[V3], @@ -253,11 +364,10 @@ impl Scene { lo[0].is_finite().then_some((lo, hi)) } - /// Frame the geometry, orbit to `camera`, light, cull, and depth-sort. - pub fn project(&self, camera: Camera, width: f32, height: f32) -> Projection { - let Some((lo, hi)) = self.bounds() else { - return Projection::default(); - }; + /// The camera framed on this geometry and orbited to `camera`, for a + /// canvas `width` by `height`. + pub(crate) fn view_for(&self, camera: Camera, width: f32, height: f32) -> Option { + let (lo, hi) = self.bounds()?; let target = scale(add(lo, hi), 0.5); let half = scale(sub(hi, lo), 0.5); let radius = dot(half, half).sqrt().max(1e-3); @@ -287,18 +397,70 @@ impl Scene { * (width / 2.0 / footprint).min(height / 2.0 / tall) * distance * camera.zoom.max(0.05); - let near = distance * 0.02; - let (cx, cy) = (width / 2.0, height / 2.0); - let view = |p: V3| { - let d = sub(p, eye); - [dot(d, right), dot(d, up), dot(d, forward)] - }; - let screen = |v: V3| [cx + focal * v[0] / v[2], cy - focal * v[1] / v[2]]; // A soft key light from the upper left, in front of the geometry. let light = normalize(add( add(scale(right, -0.45), scale(up, 0.8)), scale(forward, -0.55), )); + Some(View { + eye, + right, + up, + forward, + focal, + near: distance * 0.02, + cx: width / 2.0, + cy: height / 2.0, + center: target, + radius, + light, + }) + } + + /// The camera framed on this geometry, for the GPU renderer. + pub fn camera_frame(&self, camera: Camera, width: f32, height: f32) -> Option { + let view = self.view_for(camera, width, height)?; + Some(CameraFrame { + eye: view.eye, + right: view.right, + up: view.up, + forward: view.forward, + focal: view.focal, + near: view.near, + center: view.center, + radius: view.radius, + }) + } + + pub fn face_views(&self) -> impl Iterator> { + self.faces.iter().map(|f| FaceView { + corners: &f.v[..f.n as usize], + normal: f.normal, + normals: f.normals.as_ref().map(|n| &n[..f.n as usize]), + color: f.color, + two_sided: f.two_sided, + material: f.material, + }) + } + + /// Lines: ends, color, width in pixels. + pub fn line_views(&self) -> impl Iterator + '_ { + self.lines.iter().map(|l| (l.a, l.b, l.color, l.width)) + } + + /// Points: center, radius in pixels, color. + pub fn point_views(&self) -> impl Iterator + '_ { + self.points.iter().map(|p| (p.at, p.radius, p.color)) + } + + /// Frame the geometry, orbit to `camera`, light, cull, and depth-sort. + pub fn project(&self, camera: Camera, width: f32, height: f32) -> Projection { + let Some(setup) = self.view_for(camera, width, height) else { + return Projection::default(); + }; + let (eye, near, radius, light) = (setup.eye, setup.near, setup.radius, setup.light); + let view = |p: V3| setup.view(p); + let screen = |v: V3| setup.screen(v); let mut sorted: Vec<(f32, Prim)> = Vec::with_capacity(self.faces.len() + self.items()); for face in &self.faces { @@ -398,6 +560,38 @@ fn v3(value: Value, what: &str) -> mlua::Result { } } +/// A primitive's material options: roughness, metal, glow, glass (true or +/// 0-1), ior, dispersion. +fn material_of(options: &Table) -> mlua::Result { + let mut m = Material::default(); + let number = |key: &str| options.raw_get::>(key); + if let Some(v) = number("roughness")? { + m.roughness = v.clamp(0.02, 1.0); + } + if let Some(v) = number("metal")? { + m.metalness = v.clamp(0.0, 1.0); + } + if let Some(v) = number("glow")? { + m.glow = v.clamp(0.0, 8.0); + } + match options.raw_get::("glass")? { + Value::Boolean(true) => { + m.glass = 1.0; + m.roughness = m.roughness.min(0.08); + } + Value::Integer(n) => m.glass = (n as f32).clamp(0.0, 1.0), + Value::Number(n) => m.glass = (n as f32).clamp(0.0, 1.0), + _ => {} + } + if let Some(v) = number("ior")? { + m.ior = v.clamp(1.0, 3.0); + } + if let Some(v) = number("dispersion")? { + m.dispersion = v.clamp(0.0, 1.0); + } + Ok(m) +} + fn color_of(options: &Table, default: Token) -> mlua::Result { Ok(parse_color(options.raw_get::("color")?) .map_err(|e| lua_error(e.to_string()))? @@ -441,6 +635,7 @@ impl UserData for Scene { fn add_methods>(methods: &mut M) { methods.add_method_mut("box", |_, s, o: Table| { + s.material = material_of(&o)?; let at = v3(o.raw_get("at")?, "box at")?; let size = match o.raw_get::("size")? { Value::Nil => [1.0; 3], @@ -515,6 +710,7 @@ impl UserData for Scene { Ok(()) }); methods.add_method_mut("sphere", |_, s, o: Table| { + s.material = material_of(&o)?; let at = v3(o.raw_get("at")?, "sphere at")?; let r = o.raw_get::>("radius")?.unwrap_or(0.5).abs(); let color = color_of(&o, Token::Accent)?; @@ -535,7 +731,6 @@ impl UserData for Scene { for j in 0..lat { for i in 0..lon { let quad = [p(i, j), p(i + 1, j), p(i + 1, j + 1), p(i, j + 1)]; - let mid = scale(quad.iter().fold([0.0; 3], |a, v| add(a, *v)), 0.25); let corners: Vec = if j == 0 { vec![quad[0], quad[2], quad[3]] } else if j == lat - 1 { @@ -543,12 +738,14 @@ impl UserData for Scene { } else { quad.to_vec() }; - s.face(&corners, color, Some(sub(mid, at)), false)?; + let normals: Vec = corners.iter().map(|c| sub(*c, at)).collect(); + s.smooth_face(&corners, &normals, color)?; } } Ok(()) }); methods.add_method_mut("cylinder", |_, s, o: Table| { + s.material = material_of(&o)?; let at = v3(o.raw_get("at")?, "cylinder at")?; let r = o.raw_get::>("radius")?.unwrap_or(0.5).abs(); let h = o.raw_get::>("height")?.unwrap_or(1.0); @@ -563,8 +760,11 @@ impl UserData for Scene { }; for i in 0..n { let quad = [ring(i, 0.0), ring(i + 1, 0.0), ring(i + 1, h), ring(i, h)]; - let mid = scale(add(quad[0], quad[2]), 0.5); - s.face(&quad, color, Some(sub(mid, [at[0], mid[1], at[2]])), false)?; + let normals: Vec = quad + .iter() + .map(|c| [c[0] - at[0], 0.0, c[2] - at[2]]) + .collect(); + s.smooth_face(&quad, &normals, color)?; } let top: Vec = (0..n).map(|i| ring(i, h)).collect(); let bottom: Vec = (0..n).map(|i| ring(i, 0.0)).collect(); @@ -573,6 +773,7 @@ impl UserData for Scene { }); // A height field: y = f(x, z) over a grid, or `values` rows. methods.add_method_mut("surface", |_, s, o: Table| { + s.material = material_of(&o)?; let range = |key: &str| -> mlua::Result<(f32, f32)> { match o.raw_get::>(key)? { Some(t) => Ok((t.raw_get::(1)?, t.raw_get::(2)?)), @@ -653,6 +854,7 @@ impl UserData for Scene { Ok(()) }); methods.add_method_mut("mesh", |_, s, o: Table| { + s.material = material_of(&o)?; let color = color_of(&o, Token::Accent)?; let vertices: Vec = o .raw_get::
("vertices")? diff --git a/crates/visual/src/shader.rs b/crates/visual/src/shader.rs new file mode 100644 index 0000000000..84e8ca88b9 --- /dev/null +++ b/crates/visual/src/shader.rs @@ -0,0 +1,269 @@ +//! `ui.shader`: a fragment shader the desktop runs on the GPU, in the +//! convention agents already know from Shadertoy. The script supplies +//! `void mainImage(out vec4 fragColor, in vec2 fragCoord)`; Zeron wraps it in +//! a complete GLSL program with the usual inputs and validates it here, so a +//! compile error reaches the agent (with its line) before the reader sees it. +//! +//! Inputs, as in Shadertoy: `iResolution` (vec3, pixels), `iTime` and +//! `iTimeDelta` (seconds), `iFrame` (int), `iMouse` (vec4: xy while the +//! button is down, zw where it went down, negative z/w once released; +//! origin bottom-left). Zeron adds `iCamera` (vec4: yaw, pitch, zoom, 0), +//! which the reader orbits by dragging when `orbit = true`, and named number +//! inputs the script binds to its state (`inputs = { speed = s.speed }` +//! makes `speed` a float in the shader). +//! +//! Like Shadertoy's buffers, up to four extra passes (`buffers = { A, B, +//! ... }`) render first, each into its own float texture that every pass +//! reads as `iChannel0`-`iChannel3`: a later pass sees an earlier one's +//! current frame, and a pass sees its own previous frame (feedback: trails, +//! simulations, accumulation). `iChannelResolution[i]` is each channel's +//! size, and `iNoise` is a tiling 256x256 texture of independent random RGBA. + +/// At most this many named inputs (four vec4 slots). +pub const MAX_INPUTS: usize = 16; +/// Longest shader source a script may pass. +pub const MAX_SHADER_BYTES: usize = 64 * 1024; +/// Buffer passes before the image pass. +pub const MAX_BUFFERS: usize = 4; + +/// How a shader's output sits on the chat. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum Blend { + /// The alpha the shader writes (0 shows the chat through). + Alpha, + /// The output is light: brightness is opacity, so black is clear. For + /// beams, glows, particles, and anything drawn as emitted light. The + /// default: shader code habitually writes alpha 1 over a black clear + /// color, which would otherwise be a black box on the chat. + #[default] + Light, + /// Fully opaque, as on Shadertoy. + Opaque, +} + +impl Blend { + pub fn parse(name: &str) -> Option { + Some(match name { + "alpha" => Self::Alpha, + "light" | "add" | "additive" => Self::Light, + "opaque" => Self::Opaque, + _ => return None, + }) + } +} + +/// What a shader frame is drawn with, in the uniform block's layout. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +#[repr(C)] +pub struct ShaderUniforms { + /// Width, height, pixel ratio, unused. + pub resolution: [f32; 4], + pub mouse: [f32; 4], + /// Yaw, pitch, zoom, unused. + pub camera: [f32; 4], + /// Time, time delta, frame, unused. + pub time: [f32; 4], + pub inputs: [[f32; 4]; MAX_INPUTS / 4], + /// Each channel's width, height, 1, unused. + pub channels: [[f32; 4]; 4], +} + +impl ShaderUniforms { + pub fn as_bytes(&self) -> &[u8] { + // SAFETY: plain `f32` arrays with no padding. + unsafe { + std::slice::from_raw_parts( + (self as *const Self).cast::(), + std::mem::size_of::(), + ) + } + } +} + +/// The complete fragment program around `code`, and how many lines precede +/// the script's own first line (to report errors at its line numbers). +/// +/// Passes render rows bottom-up (fragCoord.y is the texture row), so a +/// buffer reads back exactly what it wrote, as on Shadertoy; the desktop +/// turns the final image upright. +pub fn program(code: &str, inputs: &[String]) -> (String, usize) { + let mut header = String::from( + "#version 450\n\ + layout(set = 0, binding = 0) uniform ZeronShader {\n\ + \x20 vec4 zeron_resolution;\n\ + \x20 vec4 iMouse;\n\ + \x20 vec4 iCamera;\n\ + \x20 vec4 zeron_time;\n\ + \x20 vec4 zeron_inputs[4];\n\ + \x20 vec4 iChannelResolution[4];\n\ + };\n\ + layout(set = 0, binding = 1) uniform texture2D zeron_channel0;\n\ + layout(set = 0, binding = 2) uniform texture2D zeron_channel1;\n\ + layout(set = 0, binding = 3) uniform texture2D zeron_channel2;\n\ + layout(set = 0, binding = 4) uniform texture2D zeron_channel3;\n\ + layout(set = 0, binding = 5) uniform texture2D zeron_noise;\n\ + layout(set = 0, binding = 6) uniform sampler zeron_clamp;\n\ + layout(set = 0, binding = 7) uniform sampler zeron_repeat;\n\ + layout(location = 0) out vec4 zeron_color;\n\ + #define iChannel0 sampler2D(zeron_channel0, zeron_clamp)\n\ + #define iChannel1 sampler2D(zeron_channel1, zeron_clamp)\n\ + #define iChannel2 sampler2D(zeron_channel2, zeron_clamp)\n\ + #define iChannel3 sampler2D(zeron_channel3, zeron_clamp)\n\ + #define iNoise sampler2D(zeron_noise, zeron_repeat)\n\ + #define iResolution (zeron_resolution.xyz)\n\ + #define iTime (zeron_time.x)\n\ + #define iTimeDelta (zeron_time.y)\n\ + #define iFrame (int(zeron_time.z))\n", + ); + for (i, name) in inputs.iter().enumerate().take(MAX_INPUTS) { + let lane = ["x", "y", "z", "w"][i % 4]; + header.push_str(&format!("#define {name} (zeron_inputs[{}].{lane})\n", i / 4)); + } + let lines = header.lines().count(); + // Shadertoy sources often carry an ES precision statement; GLSL 4.50 + // has no use for it. + let body: String = code + .lines() + .map(|line| { + if line.trim_start().starts_with("precision ") { + "" + } else { + line + } + }) + .collect::>() + .join("\n"); + let program = format!( + "{header}{body}\n\ + void main() {{\n\ + \x20 vec4 color = vec4(0.0, 0.0, 0.0, 1.0);\n\ + \x20 mainImage(color, gl_FragCoord.xy);\n\ + \x20 zeron_color = color;\n\ + }}\n" + ); + (program, lines) +} + +/// Whether `name` can be a shader input: a GLSL identifier that does not +/// shadow the built-in inputs. +pub fn valid_input_name(name: &str) -> bool { + let mut chars = name.chars(); + let starts = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_'); + starts + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') + && !name.starts_with("gl_") + && !name.starts_with("zeron_") + && !matches!( + name, + "iResolution" + | "iTime" + | "iTimeDelta" + | "iFrame" + | "iMouse" + | "iCamera" + | "iChannel0" + | "iChannel1" + | "iChannel2" + | "iChannel3" + | "iChannelResolution" + | "iNoise" + | "main" + ) +} + +/// Compile `code` as the GPU will, and report the first problem at the +/// script's own line numbers. +pub fn validate(code: &str, inputs: &[String]) -> Result<(), String> { + if code.len() > MAX_SHADER_BYTES { + return Err(format!("a shader is at most {MAX_SHADER_BYTES} bytes")); + } + if !code.contains("mainImage") { + return Err( + "a shader defines void mainImage(out vec4 fragColor, in vec2 fragCoord)".into(), + ); + } + let (source, offset) = program(code, inputs); + let module = naga::front::glsl::Frontend::default() + .parse( + &naga::front::glsl::Options::from(naga::ShaderStage::Fragment), + &source, + ) + .map_err(|errors| describe(&errors.errors, &source, offset))?; + naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::all(), + ) + .validate(&module) + .map_err(|error| format!("shader: {}", error.into_inner()))?; + Ok(()) +} + +fn describe(errors: &[naga::front::glsl::Error], source: &str, offset: usize) -> String { + let Some(error) = errors.first() else { + return "shader: failed to compile".into(); + }; + let line = error.meta.location(source).line_number as usize; + if line > offset { + format!("shader line {}: {}", line - offset, error.kind) + } else { + format!("shader: {}", error.kind) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RINGS: &str = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + vec2 uv = fragCoord / iResolution.xy;\n\ + float d = length(uv - 0.5);\n\ + fragColor = vec4(vec3(0.5 + 0.5 * sin(d * 40.0 - iTime * speed)), 1.0);\n\ + }"; + + #[test] + fn a_shadertoy_style_shader_compiles_with_named_inputs() { + validate(RINGS, &["speed".into()]).unwrap(); + } + + #[test] + fn errors_point_at_the_scripts_own_line() { + let broken = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + vec2 uv = fragCoord / iResolution.xy;\n\ + fragColor = vec4(uv, nope, 1.0);\n\ + }"; + let error = validate(broken, &[]).unwrap_err(); + assert!(error.starts_with("shader line 3:"), "{error}"); + // An undeclared input is the same mistake. + assert!(validate(RINGS, &[]).is_err()); + } + + #[test] + fn passes_sample_channels_and_noise_as_on_shadertoy() { + let code = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + vec2 uv = fragCoord / iResolution.xy;\n\ + vec4 last = texture(iChannel0, uv);\n\ + vec4 n = texture(iNoise, uv * 4.0);\n\ + vec4 exact = texelFetch(iChannel1, ivec2(fragCoord), 0);\n\ + fragColor = last * 0.98 + n * 0.02 + exact * iChannelResolution[1].x * 0.0;\n\ + }"; + validate(code, &[]).unwrap(); + } + + #[test] + fn es_precision_lines_are_tolerated() { + let code = format!("precision highp float;\n{RINGS}"); + validate(&code, &["speed".into()]).unwrap(); + } + + #[test] + fn input_names_are_identifiers_that_shadow_nothing() { + assert!(valid_input_name("speed")); + assert!(valid_input_name("_k2")); + assert!(!valid_input_name("2k")); + assert!(!valid_input_name("iTime")); + assert!(!valid_input_name("gl_Position")); + assert!(!valid_input_name("zeron_x")); + } +} diff --git a/crates/visual/src/tests.rs b/crates/visual/src/tests.rs index 4f0dde7c79..f9982156f4 100644 --- a/crates/visual/src/tests.rs +++ b/crates/visual/src/tests.rs @@ -276,3 +276,64 @@ fn ticks_are_round() { assert_eq!(nice_ticks(0.0, 0.9, 4), [0.0, 0.2, 0.4, 0.6, 0.8]); assert_eq!(nice_ticks(5.0, 5.0, 4), [5.0]); } + +#[test] +fn shaders_bind_state_as_inputs_and_fail_with_their_line() { + let report = check( + r#" + local s = ui.state { speed = 1 } + return function() + return ui.column { + ui.slider { label = "Speed", value = s.speed, min = 0, max = 4, on_change = function(v) s.speed = v end }, + ui.shader { height = 240, orbit = true, inputs = { speed = s.speed }, code = [[ + void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + fragColor = vec4(uv, 0.5 + 0.5 * sin(iTime * speed + iCamera.x), 1.0); + } + ]] }, + } + end + "#, + 1, + ) + .unwrap(); + assert!(report.interactions >= 1); + + let err = check( + r#" + return function() + return ui.shader { height = 200, code = [[ +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + fragColor = vec4(missing, 1.0); +} + ]] } + end + "#, + 1, + ) + .unwrap_err(); + // Lua drops the newline after `[[`, so mainImage is line 1. + assert!(err.message.contains("shader line 2:"), "{}", err.message); +} + +#[test] +fn shaders_default_to_light_and_keep_the_old_transparent_flag() { + use crate::shader::Blend; + let blend_of = |props: &str| { + let mut visual = Visual::load( + &format!( + "return function() return ui.shader {{ height = 100, {props} code = [[void mainImage(out vec4 c, in vec2 p) {{ c = vec4(1.0); }}]] }} end" + ), + 1, + ) + .unwrap(); + match visual.render().unwrap().kind { + NodeKind::Shader(shader) => shader.blend, + _ => unreachable!(), + } + }; + assert_eq!(blend_of(""), Blend::Light); + assert_eq!(blend_of("transparent = true,"), Blend::Alpha); + assert_eq!(blend_of("transparent = false,"), Blend::Opaque); + assert_eq!(blend_of("blend = \"opaque\","), Blend::Opaque); +} diff --git a/docs/mcp.md b/docs/mcp.md index 3086517949..99dd6f9ad9 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -175,7 +175,7 @@ inline in the calling chat, above the agent's final reply. A visual is a Luau script, not a web page. The script builds its data and state once and returns a view function; the view returns a tree of `ui.*` nodes (rows, columns, grids, text, stats, tables, charts, tabs, toggles, sliders, buttons, links, 2D -canvases, and 3D scenes). Controls call the script's handlers, and Zeron renders +canvases, 3D scenes, and GPU shaders). Controls call the script's handlers, and Zeron renders the view again. `crates/visual` (`zeron-visual`) is the runtime. Scripts run in a Luau sandbox @@ -189,9 +189,19 @@ are theme tokens (`text`, `muted`, `faint`, `accent`, `on-accent`, `success`, `warning`, `danger`, `border`, `chart1`…`chart6`), text sizes are steps on the transcript's type scale, the font is always the interface font, and layout nodes have no fill, border, shadow, padding, or font props; using one is an error that -names the rule. 3D scenes are projected in the runtime: Zeron frames an orbit -camera on the geometry, lights faces with one soft light, culls and depth-sorts -them, and hands the desktop 2D polygons. +names the rule. 3D scenes are rendered in the runtime (`raster.rs`): Zeron +frames an orbit camera on the geometry, clips faces to the near plane, and +rasterizes them with a depth buffer (smooth-shaded curved solids, a sky/key/fill +light rig, depth-edge outlines, supersampled at rest), in parallel row bands, +into one transparent image. `ui.shader` takes Shadertoy-style GLSL +(`mainImage`, `iResolution`, `iTime`, `iMouse`, plus `iCamera` for an orbit +camera, named float inputs bound to the script's state, `iNoise`, and up to four +Shadertoy-style buffer passes read as `iChannel0`-`3`, with feedback from their +own previous frame); `zeron_visual::shader` wraps each pass in a GLSL 4.50 +program and validates it with naga, so a compile error reaches the agent with +its pass and line. A shader composites onto the chat by `blend` (`alpha`, +`light` for emitted light where black is clear, or `opaque`) with optional +bloom, so it never sits in a black box. The engine runs every script before showing it (`zeron_visual::check`: setup, the first render, every canvas and scene draw, then each tab, toggle, slider @@ -205,8 +215,10 @@ clients see an empty part. The desktop (`crates/ui/src/visuals.rs`) renders the tree with the transcript's own GPUI elements: markdown's table treatment, the settings tab, switch, and -button idioms, and canvases and scenes painted as GPUI paths and shaped text in -theme colors. Nothing foreign is hosted, so a visual scrolls, clips, blurs under +button idioms, canvases as GPUI paths and shaped text in theme colors, and +scenes and shaders as images. Shaders run offscreen on the GPU through wgpu +(`crates/visual-gpu`, one shared device, pipelines cached by source) and are +read back like a scene's image. Nothing foreign is hosted, so a visual scrolls, clips, blurs under the glass composer, and takes input like any other row, identically on Linux, macOS, and Windows, and it follows theme, appearance, and text-size changes without rerunning the script. From d9db3e4ef496d47f0e1f2d3a209670b4b42e9022 Mon Sep 17 00:00:00 2001 From: Wing <44917547+wingleeio@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:28:02 -0700 Subject: [PATCH 3/4] Add visual assets and iOS visuals, and harden untrusted visual input Assets: show_visual takes files (glTF models, PNG/JPEG/WebP/HDR images) from the agent's working directory. The engine checks and stores them beside the script; scenes gain s:model, textures, environment maps and sprites, and shaders get iTexture0-3. Limits (16 files, 8 MB each, 10 MB together) fit the MCP server's one WebSocket message to the engine. Rendering: GPU scenes and shaders fade out toward their edges; plain translucent surfaces blend unbent (only glass refracts) and, with sprites, draw before glass so it refracts them; bloom composites correctly in alpha mode; shader images sample upright as on Shadertoy. iOS: each visual gets a Rust worker thread in zeron-mobile that runs the script, lays the tree out, draws canvases and charts, and renders scenes and shaders with wgpu; the transcript takes each visual's measured height, and VisualView only paints what arrives and forwards touches. The demo gains an "Agent visuals" chat (ZERON_DEMO_VISUALS can point it at a desktop's visuals). Hardening (security and code review): - glTF: images stay encoded and are size-checked when drawn; accessor shapes are validated; node walks are bounded; parsing can't panic. - Images: a pixel budget besides the side limit; textures larger than the GPU allows are refused. - wgpu errors are caught or logged, never a panic; the texture cache is keyed by content, bounded, and decodes outside its lock. - The engine checks for a running turn first and removes partial writes; asset names may not differ only by case; the MCP server reads only regular files, bounded. - Desktop: released visuals free their GPU images; iTime and iMouse follow the docs; no second render when a drag ends. - iOS: fonts are shared, not leaked per visual; offscreen and backgrounded visuals pause; commands see the tree the last one left; failed loads retry. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 59 + apps/ios/README.md | 6 + .../ios/Zeron/Core/Generated/zeron_core.swift | 7063 ++++++++++------- .../ios/Zeron/Session/CoreSessionSource.swift | 13 +- apps/ios/Zeron/Transcript/RowView.swift | 25 +- .../Zeron/Transcript/TranscriptListView.swift | 15 + apps/ios/Zeron/Transcript/VisualView.swift | 525 ++ apps/ios/ZeronUITests/VisualFlowTests.swift | 51 + crates/client/Cargo.toml | 1 + crates/client/src/demo/fixtures.rs | 10 + crates/client/src/demo/mod.rs | 4 + crates/client/src/demo/transcripts.rs | 44 + crates/client/src/demo/visuals.rs | 163 + crates/client/tests/demo.rs | 10 +- crates/engine/src/rpc.rs | 89 +- crates/engine/src/sessions.rs | 5 + crates/engine/src/uploads.rs | 75 + crates/mcp/Cargo.toml | 1 + crates/mcp/src/tools.rs | 103 +- crates/mobile/Cargo.toml | 2 + crates/mobile/src/layout/display.rs | 2 + crates/mobile/src/layout/mod.rs | 110 +- crates/mobile/src/layout/rows.rs | 55 +- crates/mobile/src/layout/style.rs | 4 +- crates/mobile/src/layout/tests.rs | 44 +- crates/mobile/src/lib.rs | 1 + crates/mobile/src/visual/chart.rs | 658 ++ crates/mobile/src/visual/layout.rs | 1107 +++ crates/mobile/src/visual/mod.rs | 1468 ++++ crates/mobile/src/visual/paint.rs | 388 + crates/mobile/src/visual/palette.rs | 198 + crates/mobile/src/visual/tests.rs | 507 ++ crates/proto/src/agent.rs | 21 + crates/text/src/font.rs | 10 +- crates/ui/src/visuals.rs | 183 +- crates/visual-gpu/Cargo.toml | 4 + crates/visual-gpu/examples/render_shader.rs | 1 + crates/visual-gpu/src/lib.rs | 74 +- crates/visual-gpu/src/scene.rs | 716 +- crates/visual-gpu/src/shader.rs | 297 +- crates/visual-gpu/src/textures.rs | 386 + crates/visual/Cargo.toml | 6 + crates/visual/src/assets.rs | 82 + crates/visual/src/draw.rs | 23 +- crates/visual/src/image.rs | 110 + crates/visual/src/lib.rs | 91 +- crates/visual/src/model.rs | 458 ++ crates/visual/src/node.rs | 25 +- crates/visual/src/raster.rs | 100 +- crates/visual/src/scene.rs | 392 +- crates/visual/src/shader.rs | 22 +- crates/visual/src/style.rs | 3 + crates/visual/src/tests.rs | 102 + docs/mcp.md | 15 +- 54 files changed, 12987 insertions(+), 2940 deletions(-) create mode 100644 apps/ios/Zeron/Transcript/VisualView.swift create mode 100644 apps/ios/ZeronUITests/VisualFlowTests.swift create mode 100644 crates/client/src/demo/visuals.rs create mode 100644 crates/mobile/src/visual/chart.rs create mode 100644 crates/mobile/src/visual/layout.rs create mode 100644 crates/mobile/src/visual/mod.rs create mode 100644 crates/mobile/src/visual/paint.rs create mode 100644 crates/mobile/src/visual/palette.rs create mode 100644 crates/mobile/src/visual/tests.rs create mode 100644 crates/visual-gpu/src/textures.rs create mode 100644 crates/visual/src/assets.rs create mode 100644 crates/visual/src/image.rs create mode 100644 crates/visual/src/model.rs diff --git a/Cargo.lock b/Cargo.lock index 7c3a813440..4ed010ad9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3732,6 +3732,45 @@ dependencies = [ "web-sys", ] +[[package]] +name = "gltf" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ce1918195723ce6ac74e80542c5a96a40c2b26162c1957a5cd70799b8cacf7" +dependencies = [ + "base64 0.13.1", + "byteorder", + "gltf-json", + "image", + "lazy_static", + "serde_json", + "urlencoding", +] + +[[package]] +name = "gltf-derive" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14070e711538afba5d6c807edb74bcb84e5dbb9211a3bf5dea0dfab5b24f4c51" +dependencies = [ + "inflections", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "gltf-json" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6176f9d60a7eab0a877e8e96548605dedbde9190a7ae1e80bbcc1c9af03ab14" +dependencies = [ + "gltf-derive", + "serde", + "serde_derive", + "serde_json", +] + [[package]] name = "glutin_wgl_sys" version = "0.6.1" @@ -4871,6 +4910,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + [[package]] name = "inotify" version = "0.10.2" @@ -11496,6 +11541,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "usvg" version = "0.46.0" @@ -13554,6 +13605,7 @@ dependencies = [ "zeron-proto", "zeron-rpc", "zeron-sync", + "zeron-visual", "zeron-voice-session", ] @@ -13657,6 +13709,7 @@ version = "0.2.112" dependencies = [ "anyhow", "async-trait", + "base64 0.22.1", "chrono", "futures", "serde", @@ -13690,6 +13743,8 @@ dependencies = [ "zeron-syntax", "zeron-text", "zeron-veil", + "zeron-visual", + "zeron-visual-gpu", "zeron-voice-session", ] @@ -13947,6 +14002,9 @@ version = "0.2.112" name = "zeron-visual" version = "0.2.112" dependencies = [ + "base64 0.22.1", + "gltf", + "image", "mlua", "naga", "thiserror 2.0.20", @@ -13956,6 +14014,7 @@ dependencies = [ name = "zeron-visual-gpu" version = "0.2.112" dependencies = [ + "image", "pollster 0.4.0", "tracing", "wgpu", diff --git a/apps/ios/README.md b/apps/ios/README.md index b83117fcae..5c6e3eabe6 100644 --- a/apps/ios/README.md +++ b/apps/ios/README.md @@ -70,6 +70,12 @@ ZeronShared/ Compiled into the app and the extension: the activity's paints runs with CoreText at the Rust coordinates — measurement and rendering can't disagree. Display models for rows beyond the viewport are prefetched off the main thread. +5. Agent visuals (`show_visual` Luau scripts) get a worker thread each + (`crates/mobile/src/visual`): it runs the script, lays the node tree out, + draws canvases and charts as vector paint, renders scenes and shaders with + wgpu, and reports its height back to the row. `VisualView` paints what + arrives (rasterized on the worker thread) and forwards touches; the demo's + "Agent visuals" chat (`-demo -route chat:chat-visuals`) has one of each. ## Launch arguments diff --git a/apps/ios/Zeron/Core/Generated/zeron_core.swift b/apps/ios/Zeron/Core/Generated/zeron_core.swift index 55e89c2ef4..3d9f88b446 100644 --- a/apps/ios/Zeron/Core/Generated/zeron_core.swift +++ b/apps/ios/Zeron/Core/Generated/zeron_core.swift @@ -3677,6 +3677,12 @@ public protocol TranscriptViewProtocol: AnyObject, Sendable { func setViewport(width: Float, textScale: Float) + /** + * Where this transcript's visuals load their scripts from (the chat's + * host device), and the screen's pixel density for their frames. + */ + func setVisualSource(client: CoreClient, deviceId: String, pixelScale: Float) + /** * Expand/collapse a disclosure (tool group, long user message). */ @@ -3687,6 +3693,11 @@ public protocol TranscriptViewProtocol: AnyObject, Sendable { */ func toggleDetail(row: UInt64, detail: UInt64, `open`: Bool) + /** + * The live visual behind a `WidgetKind::Visual` (by its visual id). + */ + func visual(id: String) -> VisualSession? + } /** * One transcript's layout engine. @@ -3810,6 +3821,21 @@ open func setViewport(width: Float, textScale: Float) {try! rustCall() { FfiConverterFloat.lower(textScale),uniffiCallStatus ) } +} + + /** + * Where this transcript's visuals load their scripts from (the chat's + * host device), and the screen's pixel density for their frames. + */ +open func setVisualSource(client: CoreClient, deviceId: String, pixelScale: Float) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_transcriptview_set_visual_source( + self.uniffiCloneHandle(), + FfiConverterTypeCoreClient_lower(client), + FfiConverterString.lower(deviceId), + FfiConverterFloat.lower(pixelScale),uniffiCallStatus + ) +} } /** @@ -3838,6 +3864,19 @@ open func toggleDetail(row: UInt64, detail: UInt64, `open`: Bool) {try! rustCal } } + /** + * The live visual behind a `WidgetKind::Visual` (by its visual id). + */ +open func visual(id: String) -> VisualSession? { + return try! FfiConverterOptionTypeVisualSession.lift(try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_transcriptview_visual( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +}) +} + } @@ -4092,19 +4131,20 @@ public func FfiConverterTypeUploadProgress_lower(_ value: UploadProgress) -> UIn -public protocol VoiceCallProtocol: AnyObject, Sendable { - - /** - * Late native callbacks simply miss the retired request. No resume is possible. - */ - func completeMedia(requestId: UInt64, failure: VoiceMediaFailure?, sdp: String?, microphone: UInt16, speaker: UInt16) +/** + * Where a visual's output goes (called on its worker thread). + */ +public protocol VisualListener: AnyObject, Sendable { - func setMuted(muted: Bool) + func display(display: VisualDisplay) - func stop() + func frame(frame: SurfaceFrame) } -open class VoiceCall: VoiceCallProtocol, @unchecked Sendable { +/** + * Where a visual's output goes (called on its worker thread). + */ +open class VisualListenerImpl: VisualListener, @unchecked Sendable { fileprivate let handle: UInt64 /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. @@ -4141,7 +4181,7 @@ open class VoiceCall: VoiceCallProtocol, @unchecked Sendable { @_documentation(visibility: private) #endif public func uniffiCloneHandle() -> UInt64 { - return try! rustCall { uniffi_zeron_mobile_fn_clone_voicecall(self.handle, $0) } + return try! rustCall { uniffi_zeron_mobile_fn_clone_visuallistener(self.handle, $0) } } // No primary constructor declared for this class. @@ -4151,158 +4191,26 @@ open class VoiceCall: VoiceCallProtocol, @unchecked Sendable { return } - try! rustCall { uniffi_zeron_mobile_fn_free_voicecall(handle, $0) } + try! rustCall { uniffi_zeron_mobile_fn_free_visuallistener(handle, $0) } } - /** - * Late native callbacks simply miss the retired request. No resume is possible. - */ -open func completeMedia(requestId: UInt64, failure: VoiceMediaFailure?, sdp: String?, microphone: UInt16, speaker: UInt16) {try! rustCall() { - uniffiCallStatus in - uniffi_zeron_mobile_fn_method_voicecall_complete_media( - self.uniffiCloneHandle(), - FfiConverterUInt64.lower(requestId), - FfiConverterOptionTypeVoiceMediaFailure.lower(failure), - FfiConverterOptionString.lower(sdp), - FfiConverterUInt16.lower(microphone), - FfiConverterUInt16.lower(speaker),uniffiCallStatus - ) -} -} - -open func setMuted(muted: Bool) {try! rustCall() { +open func display(display: VisualDisplay) {try! rustCall() { uniffiCallStatus in - uniffi_zeron_mobile_fn_method_voicecall_set_muted( + uniffi_zeron_mobile_fn_method_visuallistener_display( self.uniffiCloneHandle(), - FfiConverterBool.lower(muted),uniffiCallStatus - ) -} -} - -open func stop() {try! rustCall() { - uniffiCallStatus in - uniffi_zeron_mobile_fn_method_voicecall_stop( - self.uniffiCloneHandle(),uniffiCallStatus + FfiConverterTypeVisualDisplay_lower(display),uniffiCallStatus ) } } - - -} - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public struct FfiConverterTypeVoiceCall: FfiConverter { - typealias FfiType = UInt64 - typealias SwiftType = VoiceCall - - public static func lift(_ handle: UInt64) throws -> VoiceCall { - return VoiceCall(unsafeFromHandle: handle) - } - - public static func lower(_ value: VoiceCall) -> UInt64 { - return value.uniffiCloneHandle() - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VoiceCall { - let handle: UInt64 = try readInt(&buf) - return try lift(handle) - } - - public static func write(_ value: VoiceCall, into buf: inout [UInt8]) { - writeInt(&buf, lower(value)) - } -} - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeVoiceCall_lift(_ handle: UInt64) throws -> VoiceCall { - return try FfiConverterTypeVoiceCall.lift(handle) -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeVoiceCall_lower(_ value: VoiceCall) -> UInt64 { - return FfiConverterTypeVoiceCall.lower(value) -} - - - - - - -public protocol VoiceMediaListener: AnyObject, Sendable { - - func onRequest(request: VoiceMediaRequest) - -} -open class VoiceMediaListenerImpl: VoiceMediaListener, @unchecked Sendable { - fileprivate let handle: UInt64 - - /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public struct NoHandle { - public init() {} - } - - // TODO: We'd like this to be `private` but for Swifty reasons, - // we can't implement `FfiConverter` without making this `required` and we can't - // make it `required` without making it `public`. -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - required public init(unsafeFromHandle handle: UInt64) { - self.handle = handle - } - - // This constructor can be used to instantiate a fake object. - // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. - // - // - Warning: - // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public init(noHandle: NoHandle) { - self.handle = 0 - } - -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public func uniffiCloneHandle() -> UInt64 { - return try! rustCall { uniffi_zeron_mobile_fn_clone_voicemedialistener(self.handle, $0) } - } - // No primary constructor declared for this class. - - deinit { - if handle == 0 { - // Mock objects have handle=0 don't try to free them - return - } - - try! rustCall { uniffi_zeron_mobile_fn_free_voicemedialistener(handle, $0) } - } - - - - -open func onRequest(request: VoiceMediaRequest) {try! rustCall() { +open func frame(frame: SurfaceFrame) {try! rustCall() { uniffiCallStatus in - uniffi_zeron_mobile_fn_method_voicemedialistener_on_request( + uniffi_zeron_mobile_fn_method_visuallistener_frame( self.uniffiCloneHandle(), - FfiConverterTypeVoiceMediaRequest_lower(request),uniffiCallStatus + FfiConverterTypeSurfaceFrame_lower(frame),uniffiCallStatus ) } } @@ -4314,40 +4222,64 @@ open func onRequest(request: VoiceMediaRequest) {try! rustCall() { // Put the implementation in a struct so we don't pollute the top-level namespace -fileprivate struct UniffiCallbackInterfaceVoiceMediaListener { +fileprivate struct UniffiCallbackInterfaceVisualListener { // Create the VTable using a series of closures. // Swift automatically converts these into C callback functions. // // Store the vtable directly. - static let vtable: UniffiVTableCallbackInterfaceVoiceMediaListener = UniffiVTableCallbackInterfaceVoiceMediaListener( + static let vtable: UniffiVTableCallbackInterfaceVisualListener = UniffiVTableCallbackInterfaceVisualListener( uniffiFree: { (uniffiHandle: UInt64) -> () in do { - try FfiConverterTypeVoiceMediaListener.handleMap.remove(handle: uniffiHandle) + try FfiConverterTypeVisualListener.handleMap.remove(handle: uniffiHandle) } catch { - print("Uniffi callback interface VoiceMediaListener: handle missing in uniffiFree") + print("Uniffi callback interface VisualListener: handle missing in uniffiFree") } }, uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in do { - return try FfiConverterTypeVoiceMediaListener.handleMap.clone(handle: uniffiHandle) + return try FfiConverterTypeVisualListener.handleMap.clone(handle: uniffiHandle) } catch { - fatalError("Uniffi callback interface VoiceMediaListener: handle missing in uniffiClone") + fatalError("Uniffi callback interface VisualListener: handle missing in uniffiClone") } }, - onRequest: { ( + display: { ( uniffiHandle: UInt64, - request: RustBuffer, + display: RustBuffer, uniffiOutReturn: UnsafeMutableRawPointer, uniffiCallStatus: UnsafeMutablePointer ) in let makeCall = { () throws -> () in - guard let uniffiObj = try? FfiConverterTypeVoiceMediaListener.handleMap.get(handle: uniffiHandle) else { + guard let uniffiObj = try? FfiConverterTypeVisualListener.handleMap.get(handle: uniffiHandle) else { throw UniffiInternalError.unexpectedStaleHandle } - return uniffiObj.onRequest( - request: try FfiConverterTypeVoiceMediaRequest_lift(request) + return uniffiObj.display( + display: try FfiConverterTypeVisualDisplay_lift(display) + ) + } + + + let writeReturn = { () } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + }, + frame: { ( + uniffiHandle: UInt64, + frame: RustBuffer, + uniffiOutReturn: UnsafeMutableRawPointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> () in + guard let uniffiObj = try? FfiConverterTypeVisualListener.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.frame( + frame: try FfiConverterTypeSurfaceFrame_lift(frame) ) } @@ -4367,39 +4299,39 @@ fileprivate struct UniffiCallbackInterfaceVoiceMediaListener { // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. // This is safe because the pointee is initialized once during static init // and never mutated by either side of the FFI. Its fields are C function pointers. - nonisolated(unsafe) static let vtablePtr: UnsafePointer = { - let ptr = UnsafeMutablePointer.allocate(capacity: 1) + nonisolated(unsafe) static let vtablePtr: UnsafePointer = { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) ptr.initialize(to: vtable) return UnsafePointer(ptr) }() } -private func uniffiCallbackInitVoiceMediaListener() { - uniffi_zeron_mobile_fn_init_callback_vtable_voicemedialistener(UniffiCallbackInterfaceVoiceMediaListener.vtablePtr) +private func uniffiCallbackInitVisualListener() { + uniffi_zeron_mobile_fn_init_callback_vtable_visuallistener(UniffiCallbackInterfaceVisualListener.vtablePtr) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeVoiceMediaListener: FfiConverter { - fileprivate static let handleMap = UniffiHandleMap() +public struct FfiConverterTypeVisualListener: FfiConverter { + fileprivate static let handleMap = UniffiHandleMap() typealias FfiType = UInt64 - typealias SwiftType = VoiceMediaListener + typealias SwiftType = VisualListener - public static func lift(_ handle: UInt64) throws -> VoiceMediaListener { + public static func lift(_ handle: UInt64) throws -> VisualListener { if ((handle & 1) == 0) { // Rust-generated handle, construct a new class that uses the handle to implement the // interface - return VoiceMediaListenerImpl(unsafeFromHandle: handle) + return VisualListenerImpl(unsafeFromHandle: handle) } else { // Swift-generated handle, get the object from the handle map return try handleMap.remove(handle: handle) } } - public static func lower(_ value: VoiceMediaListener) -> UInt64 { - if let rustImpl = value as? VoiceMediaListenerImpl { + public static func lower(_ value: VisualListener) -> UInt64 { + if let rustImpl = value as? VisualListenerImpl { // Rust-implemented object. Clone the handle and return it return rustImpl.uniffiCloneHandle() } else { @@ -4408,12 +4340,12 @@ public struct FfiConverterTypeVoiceMediaListener: FfiConverter { } } - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VoiceMediaListener { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualListener { let handle: UInt64 = try readInt(&buf) return try lift(handle) } - public static func write(_ value: VoiceMediaListener, into buf: inout [UInt8]) { + public static func write(_ value: VisualListener, into buf: inout [UInt8]) { writeInt(&buf, lower(value)) } } @@ -4422,15 +4354,15 @@ public struct FfiConverterTypeVoiceMediaListener: FfiConverter { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeVoiceMediaListener_lift(_ handle: UInt64) throws -> VoiceMediaListener { - return try FfiConverterTypeVoiceMediaListener.lift(handle) +public func FfiConverterTypeVisualListener_lift(_ handle: UInt64) throws -> VisualListener { + return try FfiConverterTypeVisualListener.lift(handle) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeVoiceMediaListener_lower(_ value: VoiceMediaListener) -> UInt64 { - return FfiConverterTypeVoiceMediaListener.lower(value) +public func FfiConverterTypeVisualListener_lower(_ value: VisualListener) -> UInt64 { + return FfiConverterTypeVisualListener.lower(value) } @@ -4438,17 +4370,45 @@ public func FfiConverterTypeVoiceMediaListener_lower(_ value: VoiceMediaListener -public protocol VoiceSessionListener: AnyObject, Sendable { +/** + * One visual: the handle the platform paints and touches through. + */ +public protocol VisualSessionProtocol: AnyObject, Sendable { - func onVoiceState(state: VoiceCallState) + /** + * Tap a button, tab, toggle or tappable stack. + */ + func activate(control: UInt64) /** - * Terminal, exactly once. `None` is an orderly end. + * Start delivering displays and frames (replacing any listener). */ - func onVoiceClosed(reason: VoiceEndReason?) + func attach(listener: VisualListener, dark: Bool, reduceMotion: Bool) + + /** + * Stop delivering (offscreen); animation pauses. + */ + func detach() + + /** + * A touch on a surface, in its coordinates. + */ + func pointer(surface: UInt64, phase: PointerPhase, x: Float, y: Float) + + func setAppearance(dark: Bool) + + /** + * Move a slider to `fraction` (0–1) of its track. + */ + func slide(control: UInt64, fraction: Float) + + func title() -> String } -open class VoiceSessionListenerImpl: VoiceSessionListener, @unchecked Sendable { +/** + * One visual: the handle the platform paints and touches through. + */ +open class VisualSession: VisualSessionProtocol, @unchecked Sendable { fileprivate let handle: UInt64 /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. @@ -4485,7 +4445,7 @@ open class VoiceSessionListenerImpl: VoiceSessionListener, @unchecked Sendable { @_documentation(visibility: private) #endif public func uniffiCloneHandle() -> UInt64 { - return try! rustCall { uniffi_zeron_mobile_fn_clone_voicesessionlistener(self.handle, $0) } + return try! rustCall { uniffi_zeron_mobile_fn_clone_visualsession(self.handle, $0) } } // No primary constructor declared for this class. @@ -4495,165 +4455,1220 @@ open class VoiceSessionListenerImpl: VoiceSessionListener, @unchecked Sendable { return } - try! rustCall { uniffi_zeron_mobile_fn_free_voicesessionlistener(handle, $0) } + try! rustCall { uniffi_zeron_mobile_fn_free_visualsession(handle, $0) } } -open func onVoiceState(state: VoiceCallState) {try! rustCall() { + /** + * Tap a button, tab, toggle or tappable stack. + */ +open func activate(control: UInt64) {try! rustCall() { uniffiCallStatus in - uniffi_zeron_mobile_fn_method_voicesessionlistener_on_voice_state( + uniffi_zeron_mobile_fn_method_visualsession_activate( self.uniffiCloneHandle(), - FfiConverterTypeVoiceCallState_lower(state),uniffiCallStatus + FfiConverterUInt64.lower(control),uniffiCallStatus ) } } /** - * Terminal, exactly once. `None` is an orderly end. + * Start delivering displays and frames (replacing any listener). */ -open func onVoiceClosed(reason: VoiceEndReason?) {try! rustCall() { +open func attach(listener: VisualListener, dark: Bool, reduceMotion: Bool) {try! rustCall() { uniffiCallStatus in - uniffi_zeron_mobile_fn_method_voicesessionlistener_on_voice_closed( + uniffi_zeron_mobile_fn_method_visualsession_attach( self.uniffiCloneHandle(), - FfiConverterOptionTypeVoiceEndReason.lower(reason),uniffiCallStatus + FfiConverterTypeVisualListener_lower(listener), + FfiConverterBool.lower(dark), + FfiConverterBool.lower(reduceMotion),uniffiCallStatus ) } } + /** + * Stop delivering (offscreen); animation pauses. + */ +open func detach() {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_visualsession_detach( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * A touch on a surface, in its coordinates. + */ +open func pointer(surface: UInt64, phase: PointerPhase, x: Float, y: Float) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_visualsession_pointer( + self.uniffiCloneHandle(), + FfiConverterUInt64.lower(surface), + FfiConverterTypePointerPhase_lower(phase), + FfiConverterFloat.lower(x), + FfiConverterFloat.lower(y),uniffiCallStatus + ) +} +} + +open func setAppearance(dark: Bool) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_visualsession_set_appearance( + self.uniffiCloneHandle(), + FfiConverterBool.lower(dark),uniffiCallStatus + ) +} +} + + /** + * Move a slider to `fraction` (0–1) of its track. + */ +open func slide(control: UInt64, fraction: Float) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_visualsession_slide( + self.uniffiCloneHandle(), + FfiConverterUInt64.lower(control), + FfiConverterFloat.lower(fraction),uniffiCallStatus + ) +} +} + +open func title() -> String { + return try! FfiConverterString.lift(try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_visualsession_title( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeVisualSession: FfiConverter { + typealias FfiType = UInt64 + typealias SwiftType = VisualSession -// Put the implementation in a struct so we don't pollute the top-level namespace -fileprivate struct UniffiCallbackInterfaceVoiceSessionListener { + public static func lift(_ handle: UInt64) throws -> VisualSession { + return VisualSession(unsafeFromHandle: handle) + } - // Create the VTable using a series of closures. - // Swift automatically converts these into C callback functions. + public static func lower(_ value: VisualSession) -> UInt64 { + return value.uniffiCloneHandle() + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualSession { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: VisualSession, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVisualSession_lift(_ handle: UInt64) throws -> VisualSession { + return try FfiConverterTypeVisualSession.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVisualSession_lower(_ value: VisualSession) -> UInt64 { + return FfiConverterTypeVisualSession.lower(value) +} + + + + + + +public protocol VoiceCallProtocol: AnyObject, Sendable { + + /** + * Late native callbacks simply miss the retired request. No resume is possible. + */ + func completeMedia(requestId: UInt64, failure: VoiceMediaFailure?, sdp: String?, microphone: UInt16, speaker: UInt16) + + func setMuted(muted: Bool) + + func stop() + +} +open class VoiceCall: VoiceCallProtocol, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. // - // Store the vtable directly. - static let vtable: UniffiVTableCallbackInterfaceVoiceSessionListener = UniffiVTableCallbackInterfaceVoiceSessionListener( - uniffiFree: { (uniffiHandle: UInt64) -> () in - do { - try FfiConverterTypeVoiceSessionListener.handleMap.remove(handle: uniffiHandle) - } catch { - print("Uniffi callback interface VoiceSessionListener: handle missing in uniffiFree") - } - }, - uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in - do { - return try FfiConverterTypeVoiceSessionListener.handleMap.clone(handle: uniffiHandle) - } catch { - fatalError("Uniffi callback interface VoiceSessionListener: handle missing in uniffiClone") - } - }, - onVoiceState: { ( - uniffiHandle: UInt64, - state: RustBuffer, - uniffiOutReturn: UnsafeMutableRawPointer, - uniffiCallStatus: UnsafeMutablePointer - ) in - let makeCall = { - () throws -> () in - guard let uniffiObj = try? FfiConverterTypeVoiceSessionListener.handleMap.get(handle: uniffiHandle) else { - throw UniffiInternalError.unexpectedStaleHandle - } - return uniffiObj.onVoiceState( - state: try FfiConverterTypeVoiceCallState_lift(state) - ) - } + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } - - let writeReturn = { () } - uniffiTraitInterfaceCall( - callStatus: uniffiCallStatus, - makeCall: makeCall, - writeReturn: writeReturn - ) - }, - onVoiceClosed: { ( - uniffiHandle: UInt64, - reason: RustBuffer, - uniffiOutReturn: UnsafeMutableRawPointer, - uniffiCallStatus: UnsafeMutablePointer - ) in - let makeCall = { - () throws -> () in - guard let uniffiObj = try? FfiConverterTypeVoiceSessionListener.handleMap.get(handle: uniffiHandle) else { - throw UniffiInternalError.unexpectedStaleHandle - } - return uniffiObj.onVoiceClosed( - reason: try FfiConverterOptionTypeVoiceEndReason.lift(reason) - ) - } +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_zeron_mobile_fn_clone_voicecall(self.handle, $0) } + } + // No primary constructor declared for this class. - - let writeReturn = { () } - uniffiTraitInterfaceCall( - callStatus: uniffiCallStatus, - makeCall: makeCall, - writeReturn: writeReturn - ) + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return } - ) - // Rust stores this pointer for future callback invocations, so it must live - // for the process lifetime (not just for the init function call). - // - // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. - // This is safe because the pointee is initialized once during static init - // and never mutated by either side of the FFI. Its fields are C function pointers. - nonisolated(unsafe) static let vtablePtr: UnsafePointer = { - let ptr = UnsafeMutablePointer.allocate(capacity: 1) - ptr.initialize(to: vtable) - return UnsafePointer(ptr) - }() + try! rustCall { uniffi_zeron_mobile_fn_free_voicecall(handle, $0) } + } + + + + + /** + * Late native callbacks simply miss the retired request. No resume is possible. + */ +open func completeMedia(requestId: UInt64, failure: VoiceMediaFailure?, sdp: String?, microphone: UInt16, speaker: UInt16) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_voicecall_complete_media( + self.uniffiCloneHandle(), + FfiConverterUInt64.lower(requestId), + FfiConverterOptionTypeVoiceMediaFailure.lower(failure), + FfiConverterOptionString.lower(sdp), + FfiConverterUInt16.lower(microphone), + FfiConverterUInt16.lower(speaker),uniffiCallStatus + ) +} +} + +open func setMuted(muted: Bool) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_voicecall_set_muted( + self.uniffiCloneHandle(), + FfiConverterBool.lower(muted),uniffiCallStatus + ) +} +} + +open func stop() {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_voicecall_stop( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} } + -private func uniffiCallbackInitVoiceSessionListener() { - uniffi_zeron_mobile_fn_init_callback_vtable_voicesessionlistener(UniffiCallbackInterfaceVoiceSessionListener.vtablePtr) + } + #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeVoiceSessionListener: FfiConverter { - fileprivate static let handleMap = UniffiHandleMap() - +public struct FfiConverterTypeVoiceCall: FfiConverter { typealias FfiType = UInt64 - typealias SwiftType = VoiceSessionListener + typealias SwiftType = VoiceCall - public static func lift(_ handle: UInt64) throws -> VoiceSessionListener { - if ((handle & 1) == 0) { - // Rust-generated handle, construct a new class that uses the handle to implement the - // interface - return VoiceSessionListenerImpl(unsafeFromHandle: handle) - } else { - // Swift-generated handle, get the object from the handle map + public static func lift(_ handle: UInt64) throws -> VoiceCall { + return VoiceCall(unsafeFromHandle: handle) + } + + public static func lower(_ value: VoiceCall) -> UInt64 { + return value.uniffiCloneHandle() + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VoiceCall { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: VoiceCall, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVoiceCall_lift(_ handle: UInt64) throws -> VoiceCall { + return try FfiConverterTypeVoiceCall.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVoiceCall_lower(_ value: VoiceCall) -> UInt64 { + return FfiConverterTypeVoiceCall.lower(value) +} + + + + + + +public protocol VoiceMediaListener: AnyObject, Sendable { + + func onRequest(request: VoiceMediaRequest) + +} +open class VoiceMediaListenerImpl: VoiceMediaListener, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_zeron_mobile_fn_clone_voicemedialistener(self.handle, $0) } + } + // No primary constructor declared for this class. + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_zeron_mobile_fn_free_voicemedialistener(handle, $0) } + } + + + + +open func onRequest(request: VoiceMediaRequest) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_voicemedialistener_on_request( + self.uniffiCloneHandle(), + FfiConverterTypeVoiceMediaRequest_lower(request),uniffiCallStatus + ) +} +} + + + +} + + + +// Put the implementation in a struct so we don't pollute the top-level namespace +fileprivate struct UniffiCallbackInterfaceVoiceMediaListener { + + // Create the VTable using a series of closures. + // Swift automatically converts these into C callback functions. + // + // Store the vtable directly. + static let vtable: UniffiVTableCallbackInterfaceVoiceMediaListener = UniffiVTableCallbackInterfaceVoiceMediaListener( + uniffiFree: { (uniffiHandle: UInt64) -> () in + do { + try FfiConverterTypeVoiceMediaListener.handleMap.remove(handle: uniffiHandle) + } catch { + print("Uniffi callback interface VoiceMediaListener: handle missing in uniffiFree") + } + }, + uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in + do { + return try FfiConverterTypeVoiceMediaListener.handleMap.clone(handle: uniffiHandle) + } catch { + fatalError("Uniffi callback interface VoiceMediaListener: handle missing in uniffiClone") + } + }, + onRequest: { ( + uniffiHandle: UInt64, + request: RustBuffer, + uniffiOutReturn: UnsafeMutableRawPointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> () in + guard let uniffiObj = try? FfiConverterTypeVoiceMediaListener.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.onRequest( + request: try FfiConverterTypeVoiceMediaRequest_lift(request) + ) + } + + + let writeReturn = { () } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + } + ) + + // Rust stores this pointer for future callback invocations, so it must live + // for the process lifetime (not just for the init function call). + // + // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. + // This is safe because the pointee is initialized once during static init + // and never mutated by either side of the FFI. Its fields are C function pointers. + nonisolated(unsafe) static let vtablePtr: UnsafePointer = { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: vtable) + return UnsafePointer(ptr) + }() +} + +private func uniffiCallbackInitVoiceMediaListener() { + uniffi_zeron_mobile_fn_init_callback_vtable_voicemedialistener(UniffiCallbackInterfaceVoiceMediaListener.vtablePtr) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeVoiceMediaListener: FfiConverter { + fileprivate static let handleMap = UniffiHandleMap() + + typealias FfiType = UInt64 + typealias SwiftType = VoiceMediaListener + + public static func lift(_ handle: UInt64) throws -> VoiceMediaListener { + if ((handle & 1) == 0) { + // Rust-generated handle, construct a new class that uses the handle to implement the + // interface + return VoiceMediaListenerImpl(unsafeFromHandle: handle) + } else { + // Swift-generated handle, get the object from the handle map return try handleMap.remove(handle: handle) } } - public static func lower(_ value: VoiceSessionListener) -> UInt64 { - if let rustImpl = value as? VoiceSessionListenerImpl { - // Rust-implemented object. Clone the handle and return it - return rustImpl.uniffiCloneHandle() - } else { - // Swift object, generate a new vtable handle and return that. - return handleMap.insert(obj: value) - } - } + public static func lower(_ value: VoiceMediaListener) -> UInt64 { + if let rustImpl = value as? VoiceMediaListenerImpl { + // Rust-implemented object. Clone the handle and return it + return rustImpl.uniffiCloneHandle() + } else { + // Swift object, generate a new vtable handle and return that. + return handleMap.insert(obj: value) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VoiceMediaListener { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: VoiceMediaListener, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVoiceMediaListener_lift(_ handle: UInt64) throws -> VoiceMediaListener { + return try FfiConverterTypeVoiceMediaListener.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVoiceMediaListener_lower(_ value: VoiceMediaListener) -> UInt64 { + return FfiConverterTypeVoiceMediaListener.lower(value) +} + + + + + + +public protocol VoiceSessionListener: AnyObject, Sendable { + + func onVoiceState(state: VoiceCallState) + + /** + * Terminal, exactly once. `None` is an orderly end. + */ + func onVoiceClosed(reason: VoiceEndReason?) + +} +open class VoiceSessionListenerImpl: VoiceSessionListener, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_zeron_mobile_fn_clone_voicesessionlistener(self.handle, $0) } + } + // No primary constructor declared for this class. + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_zeron_mobile_fn_free_voicesessionlistener(handle, $0) } + } + + + + +open func onVoiceState(state: VoiceCallState) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_voicesessionlistener_on_voice_state( + self.uniffiCloneHandle(), + FfiConverterTypeVoiceCallState_lower(state),uniffiCallStatus + ) +} +} + + /** + * Terminal, exactly once. `None` is an orderly end. + */ +open func onVoiceClosed(reason: VoiceEndReason?) {try! rustCall() { + uniffiCallStatus in + uniffi_zeron_mobile_fn_method_voicesessionlistener_on_voice_closed( + self.uniffiCloneHandle(), + FfiConverterOptionTypeVoiceEndReason.lower(reason),uniffiCallStatus + ) +} +} + + + +} + + + +// Put the implementation in a struct so we don't pollute the top-level namespace +fileprivate struct UniffiCallbackInterfaceVoiceSessionListener { + + // Create the VTable using a series of closures. + // Swift automatically converts these into C callback functions. + // + // Store the vtable directly. + static let vtable: UniffiVTableCallbackInterfaceVoiceSessionListener = UniffiVTableCallbackInterfaceVoiceSessionListener( + uniffiFree: { (uniffiHandle: UInt64) -> () in + do { + try FfiConverterTypeVoiceSessionListener.handleMap.remove(handle: uniffiHandle) + } catch { + print("Uniffi callback interface VoiceSessionListener: handle missing in uniffiFree") + } + }, + uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in + do { + return try FfiConverterTypeVoiceSessionListener.handleMap.clone(handle: uniffiHandle) + } catch { + fatalError("Uniffi callback interface VoiceSessionListener: handle missing in uniffiClone") + } + }, + onVoiceState: { ( + uniffiHandle: UInt64, + state: RustBuffer, + uniffiOutReturn: UnsafeMutableRawPointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> () in + guard let uniffiObj = try? FfiConverterTypeVoiceSessionListener.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.onVoiceState( + state: try FfiConverterTypeVoiceCallState_lift(state) + ) + } + + + let writeReturn = { () } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + }, + onVoiceClosed: { ( + uniffiHandle: UInt64, + reason: RustBuffer, + uniffiOutReturn: UnsafeMutableRawPointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> () in + guard let uniffiObj = try? FfiConverterTypeVoiceSessionListener.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.onVoiceClosed( + reason: try FfiConverterOptionTypeVoiceEndReason.lift(reason) + ) + } + + + let writeReturn = { () } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + } + ) + + // Rust stores this pointer for future callback invocations, so it must live + // for the process lifetime (not just for the init function call). + // + // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. + // This is safe because the pointee is initialized once during static init + // and never mutated by either side of the FFI. Its fields are C function pointers. + nonisolated(unsafe) static let vtablePtr: UnsafePointer = { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: vtable) + return UnsafePointer(ptr) + }() +} + +private func uniffiCallbackInitVoiceSessionListener() { + uniffi_zeron_mobile_fn_init_callback_vtable_voicesessionlistener(UniffiCallbackInterfaceVoiceSessionListener.vtablePtr) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeVoiceSessionListener: FfiConverter { + fileprivate static let handleMap = UniffiHandleMap() + + typealias FfiType = UInt64 + typealias SwiftType = VoiceSessionListener + + public static func lift(_ handle: UInt64) throws -> VoiceSessionListener { + if ((handle & 1) == 0) { + // Rust-generated handle, construct a new class that uses the handle to implement the + // interface + return VoiceSessionListenerImpl(unsafeFromHandle: handle) + } else { + // Swift-generated handle, get the object from the handle map + return try handleMap.remove(handle: handle) + } + } + + public static func lower(_ value: VoiceSessionListener) -> UInt64 { + if let rustImpl = value as? VoiceSessionListenerImpl { + // Rust-implemented object. Clone the handle and return it + return rustImpl.uniffiCloneHandle() + } else { + // Swift object, generate a new vtable handle and return that. + return handleMap.insert(obj: value) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VoiceSessionListener { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: VoiceSessionListener, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVoiceSessionListener_lift(_ handle: UInt64) throws -> VoiceSessionListener { + return try FfiConverterTypeVoiceSessionListener.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVoiceSessionListener_lower(_ value: VoiceSessionListener) -> UInt64 { + return FfiConverterTypeVoiceSessionListener.lower(value) +} + + + + +public struct AppshotLabel: Equatable, Hashable { + public var appName: String + public var windowTitle: String? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(appName: String, windowTitle: String?) { + self.appName = appName + self.windowTitle = windowTitle + } + + + + +} + +#if compiler(>=6) +extension AppshotLabel: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeAppshotLabel: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AppshotLabel { + return + try AppshotLabel( + appName: FfiConverterString.read(from: &buf), + windowTitle: FfiConverterOptionString.read(from: &buf) + ) + } + + public static func write(_ value: AppshotLabel, into buf: inout [UInt8]) { + FfiConverterString.write(value.appName, into: &buf) + FfiConverterOptionString.write(value.windowTitle, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAppshotLabel_lift(_ buf: RustBuffer) throws -> AppshotLabel { + return try FfiConverterTypeAppshotLabel.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAppshotLabel_lower(_ value: AppshotLabel) -> RustBuffer { + return FfiConverterTypeAppshotLabel.lower(value) +} + + +public struct AuthExchange: Equatable, Hashable { + public var user: AuthUser + /** + * Unscoped pair: refresh with an organization id before use. + */ + public var tokens: AuthTokens + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(user: AuthUser, + /** + * Unscoped pair: refresh with an organization id before use. + */tokens: AuthTokens) { + self.user = user + self.tokens = tokens + } + + + + +} + +#if compiler(>=6) +extension AuthExchange: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeAuthExchange: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthExchange { + return + try AuthExchange( + user: FfiConverterTypeAuthUser.read(from: &buf), + tokens: FfiConverterTypeAuthTokens.read(from: &buf) + ) + } + + public static func write(_ value: AuthExchange, into buf: inout [UInt8]) { + FfiConverterTypeAuthUser.write(value.user, into: &buf) + FfiConverterTypeAuthTokens.write(value.tokens, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthExchange_lift(_ buf: RustBuffer) throws -> AuthExchange { + return try FfiConverterTypeAuthExchange.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthExchange_lower(_ value: AuthExchange) -> RustBuffer { + return FfiConverterTypeAuthExchange.lower(value) +} + + +public struct AuthOrg: Equatable, Hashable { + public var id: String + public var organizationId: String + public var name: String + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(id: String, organizationId: String, name: String) { + self.id = id + self.organizationId = organizationId + self.name = name + } + + + + +} + +#if compiler(>=6) +extension AuthOrg: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeAuthOrg: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthOrg { + return + try AuthOrg( + id: FfiConverterString.read(from: &buf), + organizationId: FfiConverterString.read(from: &buf), + name: FfiConverterString.read(from: &buf) + ) + } + + public static func write(_ value: AuthOrg, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterString.write(value.organizationId, into: &buf) + FfiConverterString.write(value.name, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthOrg_lift(_ buf: RustBuffer) throws -> AuthOrg { + return try FfiConverterTypeAuthOrg.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthOrg_lower(_ value: AuthOrg) -> RustBuffer { + return FfiConverterTypeAuthOrg.lower(value) +} + + +public struct AuthTokens: Equatable, Hashable { + public var accessToken: String + public var refreshToken: String + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(accessToken: String, refreshToken: String) { + self.accessToken = accessToken + self.refreshToken = refreshToken + } + + + + +} + +#if compiler(>=6) +extension AuthTokens: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeAuthTokens: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthTokens { + return + try AuthTokens( + accessToken: FfiConverterString.read(from: &buf), + refreshToken: FfiConverterString.read(from: &buf) + ) + } + + public static func write(_ value: AuthTokens, into buf: inout [UInt8]) { + FfiConverterString.write(value.accessToken, into: &buf) + FfiConverterString.write(value.refreshToken, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthTokens_lift(_ buf: RustBuffer) throws -> AuthTokens { + return try FfiConverterTypeAuthTokens.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthTokens_lower(_ value: AuthTokens) -> RustBuffer { + return FfiConverterTypeAuthTokens.lower(value) +} + + +public struct AuthUser: Equatable, Hashable { + public var id: String + public var email: String? + public var firstName: String? + public var lastName: String? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(id: String, email: String?, firstName: String?, lastName: String?) { + self.id = id + self.email = email + self.firstName = firstName + self.lastName = lastName + } + + + + +} + +#if compiler(>=6) +extension AuthUser: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeAuthUser: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthUser { + return + try AuthUser( + id: FfiConverterString.read(from: &buf), + email: FfiConverterOptionString.read(from: &buf), + firstName: FfiConverterOptionString.read(from: &buf), + lastName: FfiConverterOptionString.read(from: &buf) + ) + } + + public static func write(_ value: AuthUser, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterOptionString.write(value.email, into: &buf) + FfiConverterOptionString.write(value.firstName, into: &buf) + FfiConverterOptionString.write(value.lastName, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthUser_lift(_ buf: RustBuffer) throws -> AuthUser { + return try FfiConverterTypeAuthUser.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeAuthUser_lower(_ value: AuthUser) -> RustBuffer { + return FfiConverterTypeAuthUser.lower(value) +} + + +public struct BoxPrim: Equatable, Hashable { + public var x: Float + public var y: Float + public var w: Float + public var h: Float + public var radius: Float + public var style: BoxStyle + public var color: ColorRole + public var scroller: UInt32? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(x: Float, y: Float, w: Float, h: Float, radius: Float, style: BoxStyle, color: ColorRole, scroller: UInt32?) { + self.x = x + self.y = y + self.w = w + self.h = h + self.radius = radius + self.style = style + self.color = color + self.scroller = scroller + } + + + + +} + +#if compiler(>=6) +extension BoxPrim: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeBoxPrim: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> BoxPrim { + return + try BoxPrim( + x: FfiConverterFloat.read(from: &buf), + y: FfiConverterFloat.read(from: &buf), + w: FfiConverterFloat.read(from: &buf), + h: FfiConverterFloat.read(from: &buf), + radius: FfiConverterFloat.read(from: &buf), + style: FfiConverterTypeBoxStyle.read(from: &buf), + color: FfiConverterTypeColorRole.read(from: &buf), + scroller: FfiConverterOptionUInt32.read(from: &buf) + ) + } + + public static func write(_ value: BoxPrim, into buf: inout [UInt8]) { + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.w, into: &buf) + FfiConverterFloat.write(value.h, into: &buf) + FfiConverterFloat.write(value.radius, into: &buf) + FfiConverterTypeBoxStyle.write(value.style, into: &buf) + FfiConverterTypeColorRole.write(value.color, into: &buf) + FfiConverterOptionUInt32.write(value.scroller, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeBoxPrim_lift(_ buf: RustBuffer) throws -> BoxPrim { + return try FfiConverterTypeBoxPrim.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeBoxPrim_lower(_ value: BoxPrim) -> RustBuffer { + return FfiConverterTypeBoxPrim.lower(value) +} + + +public struct CaptionFrame: Equatable, Hashable { + /** + * The tail of the utterance to show (ellipsized in front when long). + */ + public var text: String + public var spans: [CaptionSpan] + /** + * The previous utterance while it fades out, and its opacity. + */ + public var previous: String? + public var previousAlpha: Float + /** + * Something is still fading: ask for the next frame. + */ + public var animating: Bool + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * The tail of the utterance to show (ellipsized in front when long). + */text: String, spans: [CaptionSpan], + /** + * The previous utterance while it fades out, and its opacity. + */previous: String?, previousAlpha: Float, + /** + * Something is still fading: ask for the next frame. + */animating: Bool) { + self.text = text + self.spans = spans + self.previous = previous + self.previousAlpha = previousAlpha + self.animating = animating + } + + + + +} + +#if compiler(>=6) +extension CaptionFrame: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeCaptionFrame: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CaptionFrame { + return + try CaptionFrame( + text: FfiConverterString.read(from: &buf), + spans: FfiConverterSequenceTypeCaptionSpan.read(from: &buf), + previous: FfiConverterOptionString.read(from: &buf), + previousAlpha: FfiConverterFloat.read(from: &buf), + animating: FfiConverterBool.read(from: &buf) + ) + } + + public static func write(_ value: CaptionFrame, into buf: inout [UInt8]) { + FfiConverterString.write(value.text, into: &buf) + FfiConverterSequenceTypeCaptionSpan.write(value.spans, into: &buf) + FfiConverterOptionString.write(value.previous, into: &buf) + FfiConverterFloat.write(value.previousAlpha, into: &buf) + FfiConverterBool.write(value.animating, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeCaptionFrame_lift(_ buf: RustBuffer) throws -> CaptionFrame { + return try FfiConverterTypeCaptionFrame.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeCaptionFrame_lower(_ value: CaptionFrame) -> RustBuffer { + return FfiConverterTypeCaptionFrame.lower(value) +} + + +/** + * A veiled UTF-16 range of the caption and its opacity (0..1). + */ +public struct CaptionSpan: Equatable, Hashable { + public var start: UInt32 + public var end: UInt32 + public var alpha: Float + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(start: UInt32, end: UInt32, alpha: Float) { + self.start = start + self.end = end + self.alpha = alpha + } + + - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VoiceSessionListener { - let handle: UInt64 = try readInt(&buf) - return try lift(handle) + +} + +#if compiler(>=6) +extension CaptionSpan: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeCaptionSpan: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CaptionSpan { + return + try CaptionSpan( + start: FfiConverterUInt32.read(from: &buf), + end: FfiConverterUInt32.read(from: &buf), + alpha: FfiConverterFloat.read(from: &buf) + ) } - public static func write(_ value: VoiceSessionListener, into buf: inout [UInt8]) { - writeInt(&buf, lower(value)) + public static func write(_ value: CaptionSpan, into buf: inout [UInt8]) { + FfiConverterUInt32.write(value.start, into: &buf) + FfiConverterUInt32.write(value.end, into: &buf) + FfiConverterFloat.write(value.alpha, into: &buf) } } @@ -4661,29 +5676,54 @@ public struct FfiConverterTypeVoiceSessionListener: FfiConverter { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeVoiceSessionListener_lift(_ handle: UInt64) throws -> VoiceSessionListener { - return try FfiConverterTypeVoiceSessionListener.lift(handle) +public func FfiConverterTypeCaptionSpan_lift(_ buf: RustBuffer) throws -> CaptionSpan { + return try FfiConverterTypeCaptionSpan.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeVoiceSessionListener_lower(_ value: VoiceSessionListener) -> UInt64 { - return FfiConverterTypeVoiceSessionListener.lower(value) +public func FfiConverterTypeCaptionSpan_lower(_ value: CaptionSpan) -> RustBuffer { + return FfiConverterTypeCaptionSpan.lower(value) } - - -public struct AppshotLabel: Equatable, Hashable { - public var appName: String - public var windowTitle: String? +/** + * A chat's run configuration. Ids are wire strings. + */ +public struct ChatConfig: Equatable, Hashable { + /** + * `claude-code`, `codex`, … + */ + public var harness: String + public var model: String? + /** + * `low` … `ultrathink`. + */ + public var reasoning: String? + /** + * Model option id → choice id (`contextWindow` → `1m`). + */ + public var modelOptions: [String: String] + public var sandbox: SandboxLevel // Default memberwise initializers are never public by default, so we // declare one manually. - public init(appName: String, windowTitle: String?) { - self.appName = appName - self.windowTitle = windowTitle + public init( + /** + * `claude-code`, `codex`, … + */harness: String, model: String?, + /** + * `low` … `ultrathink`. + */reasoning: String?, + /** + * Model option id → choice id (`contextWindow` → `1m`). + */modelOptions: [String: String], sandbox: SandboxLevel) { + self.harness = harness + self.model = model + self.reasoning = reasoning + self.modelOptions = modelOptions + self.sandbox = sandbox } @@ -4692,24 +5732,30 @@ public struct AppshotLabel: Equatable, Hashable { } #if compiler(>=6) -extension AppshotLabel: Sendable {} +extension ChatConfig: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeAppshotLabel: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AppshotLabel { +public struct FfiConverterTypeChatConfig: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatConfig { return - try AppshotLabel( - appName: FfiConverterString.read(from: &buf), - windowTitle: FfiConverterOptionString.read(from: &buf) + try ChatConfig( + harness: FfiConverterString.read(from: &buf), + model: FfiConverterOptionString.read(from: &buf), + reasoning: FfiConverterOptionString.read(from: &buf), + modelOptions: FfiConverterDictionaryStringString.read(from: &buf), + sandbox: FfiConverterTypeSandboxLevel.read(from: &buf) ) } - public static func write(_ value: AppshotLabel, into buf: inout [UInt8]) { - FfiConverterString.write(value.appName, into: &buf) - FfiConverterOptionString.write(value.windowTitle, into: &buf) + public static func write(_ value: ChatConfig, into buf: inout [UInt8]) { + FfiConverterString.write(value.harness, into: &buf) + FfiConverterOptionString.write(value.model, into: &buf) + FfiConverterOptionString.write(value.reasoning, into: &buf) + FfiConverterDictionaryStringString.write(value.modelOptions, into: &buf) + FfiConverterTypeSandboxLevel.write(value.sandbox, into: &buf) } } @@ -4717,33 +5763,104 @@ public struct FfiConverterTypeAppshotLabel: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAppshotLabel_lift(_ buf: RustBuffer) throws -> AppshotLabel { - return try FfiConverterTypeAppshotLabel.lift(buf) +public func FfiConverterTypeChatConfig_lift(_ buf: RustBuffer) throws -> ChatConfig { + return try FfiConverterTypeChatConfig.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAppshotLabel_lower(_ value: AppshotLabel) -> RustBuffer { - return FfiConverterTypeAppshotLabel.lower(value) +public func FfiConverterTypeChatConfig_lower(_ value: ChatConfig) -> RustBuffer { + return FfiConverterTypeChatConfig.lower(value) } -public struct AuthExchange: Equatable, Hashable { - public var user: AuthUser +/** + * Everything the composer and its status strip render. + */ +public struct ComposerState: Equatable, Hashable { + public var chatId: String + public var revision: UInt64 + public var title: String + public var host: HostInfo + public var live: LiveStatus /** - * Unscoped pair: refresh with an organization id before use. + * The shared queue, delivery order. */ - public var tokens: AuthTokens + public var queue: [QueueItem] + /** + * This device's unadopted sends, oldest first. + */ + public var pendingSends: [PendingSend] + /** + * Headline state of the oldest pending send. + */ + public var sendState: SendState? + /** + * A send now would queue rather than deliver promptly. + */ + public var deliveryDegraded: Bool + public var room: RoomState + /** + * Attachment escort progress (0..1). + */ + public var transferProgress: Double? + /** + * Answer this instead of typing. + */ + public var openInput: InputRequest? + /** + * Last queue action failure. + */ + public var queueError: String? + /** + * Scroll-to-own-send target. + */ + public var lastSubmittedMessageId: String? + public var contextUsage: ContextUsage? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(user: AuthUser, + public init(chatId: String, revision: UInt64, title: String, host: HostInfo, live: LiveStatus, /** - * Unscoped pair: refresh with an organization id before use. - */tokens: AuthTokens) { - self.user = user - self.tokens = tokens + * The shared queue, delivery order. + */queue: [QueueItem], + /** + * This device's unadopted sends, oldest first. + */pendingSends: [PendingSend], + /** + * Headline state of the oldest pending send. + */sendState: SendState?, + /** + * A send now would queue rather than deliver promptly. + */deliveryDegraded: Bool, room: RoomState, + /** + * Attachment escort progress (0..1). + */transferProgress: Double?, + /** + * Answer this instead of typing. + */openInput: InputRequest?, + /** + * Last queue action failure. + */queueError: String?, + /** + * Scroll-to-own-send target. + */lastSubmittedMessageId: String?, contextUsage: ContextUsage?) { + self.chatId = chatId + self.revision = revision + self.title = title + self.host = host + self.live = live + self.queue = queue + self.pendingSends = pendingSends + self.sendState = sendState + self.deliveryDegraded = deliveryDegraded + self.room = room + self.transferProgress = transferProgress + self.openInput = openInput + self.queueError = queueError + self.lastSubmittedMessageId = lastSubmittedMessageId + self.contextUsage = contextUsage } @@ -4752,24 +5869,50 @@ public struct AuthExchange: Equatable, Hashable { } #if compiler(>=6) -extension AuthExchange: Sendable {} +extension ComposerState: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeAuthExchange: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthExchange { +public struct FfiConverterTypeComposerState: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ComposerState { return - try AuthExchange( - user: FfiConverterTypeAuthUser.read(from: &buf), - tokens: FfiConverterTypeAuthTokens.read(from: &buf) + try ComposerState( + chatId: FfiConverterString.read(from: &buf), + revision: FfiConverterUInt64.read(from: &buf), + title: FfiConverterString.read(from: &buf), + host: FfiConverterTypeHostInfo.read(from: &buf), + live: FfiConverterTypeLiveStatus.read(from: &buf), + queue: FfiConverterSequenceTypeQueueItem.read(from: &buf), + pendingSends: FfiConverterSequenceTypePendingSend.read(from: &buf), + sendState: FfiConverterOptionTypeSendState.read(from: &buf), + deliveryDegraded: FfiConverterBool.read(from: &buf), + room: FfiConverterTypeRoomState.read(from: &buf), + transferProgress: FfiConverterOptionDouble.read(from: &buf), + openInput: FfiConverterOptionTypeInputRequest.read(from: &buf), + queueError: FfiConverterOptionString.read(from: &buf), + lastSubmittedMessageId: FfiConverterOptionString.read(from: &buf), + contextUsage: FfiConverterOptionTypeContextUsage.read(from: &buf) ) } - public static func write(_ value: AuthExchange, into buf: inout [UInt8]) { - FfiConverterTypeAuthUser.write(value.user, into: &buf) - FfiConverterTypeAuthTokens.write(value.tokens, into: &buf) + public static func write(_ value: ComposerState, into buf: inout [UInt8]) { + FfiConverterString.write(value.chatId, into: &buf) + FfiConverterUInt64.write(value.revision, into: &buf) + FfiConverterString.write(value.title, into: &buf) + FfiConverterTypeHostInfo.write(value.host, into: &buf) + FfiConverterTypeLiveStatus.write(value.live, into: &buf) + FfiConverterSequenceTypeQueueItem.write(value.queue, into: &buf) + FfiConverterSequenceTypePendingSend.write(value.pendingSends, into: &buf) + FfiConverterOptionTypeSendState.write(value.sendState, into: &buf) + FfiConverterBool.write(value.deliveryDegraded, into: &buf) + FfiConverterTypeRoomState.write(value.room, into: &buf) + FfiConverterOptionDouble.write(value.transferProgress, into: &buf) + FfiConverterOptionTypeInputRequest.write(value.openInput, into: &buf) + FfiConverterOptionString.write(value.queueError, into: &buf) + FfiConverterOptionString.write(value.lastSubmittedMessageId, into: &buf) + FfiConverterOptionTypeContextUsage.write(value.contextUsage, into: &buf) } } @@ -4777,29 +5920,49 @@ public struct FfiConverterTypeAuthExchange: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthExchange_lift(_ buf: RustBuffer) throws -> AuthExchange { - return try FfiConverterTypeAuthExchange.lift(buf) +public func FfiConverterTypeComposerState_lift(_ buf: RustBuffer) throws -> ComposerState { + return try FfiConverterTypeComposerState.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthExchange_lower(_ value: AuthExchange) -> RustBuffer { - return FfiConverterTypeAuthExchange.lower(value) +public func FfiConverterTypeComposerState_lower(_ value: ComposerState) -> RustBuffer { + return FfiConverterTypeComposerState.lower(value) } -public struct AuthOrg: Equatable, Hashable { - public var id: String - public var organizationId: String - public var name: String +public struct Connectivity: Equatable, Hashable { + public var state: ConnectivityState + /** + * Epoch ms of the next scheduled redial (countdown), if any. + */ + public var retryAtMs: Int64? + /** + * The failure that started the current outage. + */ + public var lastFailure: String? + /** + * Open chats whose room is graced-degraded. + */ + public var degradedChats: [String] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, organizationId: String, name: String) { - self.id = id - self.organizationId = organizationId - self.name = name + public init(state: ConnectivityState, + /** + * Epoch ms of the next scheduled redial (countdown), if any. + */retryAtMs: Int64?, + /** + * The failure that started the current outage. + */lastFailure: String?, + /** + * Open chats whose room is graced-degraded. + */degradedChats: [String]) { + self.state = state + self.retryAtMs = retryAtMs + self.lastFailure = lastFailure + self.degradedChats = degradedChats } @@ -4808,26 +5971,28 @@ public struct AuthOrg: Equatable, Hashable { } #if compiler(>=6) -extension AuthOrg: Sendable {} +extension Connectivity: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeAuthOrg: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthOrg { +public struct FfiConverterTypeConnectivity: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Connectivity { return - try AuthOrg( - id: FfiConverterString.read(from: &buf), - organizationId: FfiConverterString.read(from: &buf), - name: FfiConverterString.read(from: &buf) + try Connectivity( + state: FfiConverterTypeConnectivityState.read(from: &buf), + retryAtMs: FfiConverterOptionInt64.read(from: &buf), + lastFailure: FfiConverterOptionString.read(from: &buf), + degradedChats: FfiConverterSequenceString.read(from: &buf) ) } - public static func write(_ value: AuthOrg, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.organizationId, into: &buf) - FfiConverterString.write(value.name, into: &buf) + public static func write(_ value: Connectivity, into buf: inout [UInt8]) { + FfiConverterTypeConnectivityState.write(value.state, into: &buf) + FfiConverterOptionInt64.write(value.retryAtMs, into: &buf) + FfiConverterOptionString.write(value.lastFailure, into: &buf) + FfiConverterSequenceString.write(value.degradedChats, into: &buf) } } @@ -4835,27 +6000,27 @@ public struct FfiConverterTypeAuthOrg: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthOrg_lift(_ buf: RustBuffer) throws -> AuthOrg { - return try FfiConverterTypeAuthOrg.lift(buf) +public func FfiConverterTypeConnectivity_lift(_ buf: RustBuffer) throws -> Connectivity { + return try FfiConverterTypeConnectivity.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthOrg_lower(_ value: AuthOrg) -> RustBuffer { - return FfiConverterTypeAuthOrg.lower(value) +public func FfiConverterTypeConnectivity_lower(_ value: Connectivity) -> RustBuffer { + return FfiConverterTypeConnectivity.lower(value) } -public struct AuthTokens: Equatable, Hashable { - public var accessToken: String - public var refreshToken: String +public struct ContextUsage: Equatable, Hashable { + public var tokens: UInt64? + public var window: UInt64? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(accessToken: String, refreshToken: String) { - self.accessToken = accessToken - self.refreshToken = refreshToken + public init(tokens: UInt64?, window: UInt64?) { + self.tokens = tokens + self.window = window } @@ -4864,24 +6029,24 @@ public struct AuthTokens: Equatable, Hashable { } #if compiler(>=6) -extension AuthTokens: Sendable {} +extension ContextUsage: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeAuthTokens: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthTokens { +public struct FfiConverterTypeContextUsage: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ContextUsage { return - try AuthTokens( - accessToken: FfiConverterString.read(from: &buf), - refreshToken: FfiConverterString.read(from: &buf) + try ContextUsage( + tokens: FfiConverterOptionUInt64.read(from: &buf), + window: FfiConverterOptionUInt64.read(from: &buf) ) } - public static func write(_ value: AuthTokens, into buf: inout [UInt8]) { - FfiConverterString.write(value.accessToken, into: &buf) - FfiConverterString.write(value.refreshToken, into: &buf) + public static func write(_ value: ContextUsage, into buf: inout [UInt8]) { + FfiConverterOptionUInt64.write(value.tokens, into: &buf) + FfiConverterOptionUInt64.write(value.window, into: &buf) } } @@ -4889,31 +6054,70 @@ public struct FfiConverterTypeAuthTokens: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthTokens_lift(_ buf: RustBuffer) throws -> AuthTokens { - return try FfiConverterTypeAuthTokens.lift(buf) +public func FfiConverterTypeContextUsage_lift(_ buf: RustBuffer) throws -> ContextUsage { + return try FfiConverterTypeContextUsage.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthTokens_lower(_ value: AuthTokens) -> RustBuffer { - return FfiConverterTypeAuthTokens.lower(value) +public func FfiConverterTypeContextUsage_lower(_ value: ContextUsage) -> RustBuffer { + return FfiConverterTypeContextUsage.lower(value) } -public struct AuthUser: Equatable, Hashable { - public var id: String - public var email: String? - public var firstName: String? - public var lastName: String? +/** + * Static per-install configuration. + */ +public struct CoreConfig: Equatable, Hashable { + /** + * Edge base URL (`auth_production_edge_url()` for prod). + */ + public var edgeUrl: String + /** + * Writable directory for registry/doc snapshots and caches, scoped by + * the platform to the signed-in identity (sign-out wipes it). + */ + public var dataDir: String + /** + * Stable device id (`ios-xxxxxxxx`), stamped on every command/queue row. + */ + public var deviceId: String + /** + * Human name for presence/attribution ("Wing's iPhone"). + */ + public var deviceName: String + /** + * `ios` / `android`. + */ + public var platform: String + public var appVersion: String // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, email: String?, firstName: String?, lastName: String?) { - self.id = id - self.email = email - self.firstName = firstName - self.lastName = lastName + public init( + /** + * Edge base URL (`auth_production_edge_url()` for prod). + */edgeUrl: String, + /** + * Writable directory for registry/doc snapshots and caches, scoped by + * the platform to the signed-in identity (sign-out wipes it). + */dataDir: String, + /** + * Stable device id (`ios-xxxxxxxx`), stamped on every command/queue row. + */deviceId: String, + /** + * Human name for presence/attribution ("Wing's iPhone"). + */deviceName: String, + /** + * `ios` / `android`. + */platform: String, appVersion: String) { + self.edgeUrl = edgeUrl + self.dataDir = dataDir + self.deviceId = deviceId + self.deviceName = deviceName + self.platform = platform + self.appVersion = appVersion } @@ -4922,28 +6126,32 @@ public struct AuthUser: Equatable, Hashable { } #if compiler(>=6) -extension AuthUser: Sendable {} +extension CoreConfig: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeAuthUser: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> AuthUser { +public struct FfiConverterTypeCoreConfig: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CoreConfig { return - try AuthUser( - id: FfiConverterString.read(from: &buf), - email: FfiConverterOptionString.read(from: &buf), - firstName: FfiConverterOptionString.read(from: &buf), - lastName: FfiConverterOptionString.read(from: &buf) + try CoreConfig( + edgeUrl: FfiConverterString.read(from: &buf), + dataDir: FfiConverterString.read(from: &buf), + deviceId: FfiConverterString.read(from: &buf), + deviceName: FfiConverterString.read(from: &buf), + platform: FfiConverterString.read(from: &buf), + appVersion: FfiConverterString.read(from: &buf) ) } - public static func write(_ value: AuthUser, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterOptionString.write(value.email, into: &buf) - FfiConverterOptionString.write(value.firstName, into: &buf) - FfiConverterOptionString.write(value.lastName, into: &buf) + public static func write(_ value: CoreConfig, into buf: inout [UInt8]) { + FfiConverterString.write(value.edgeUrl, into: &buf) + FfiConverterString.write(value.dataDir, into: &buf) + FfiConverterString.write(value.deviceId, into: &buf) + FfiConverterString.write(value.deviceName, into: &buf) + FfiConverterString.write(value.platform, into: &buf) + FfiConverterString.write(value.appVersion, into: &buf) } } @@ -4951,39 +6159,34 @@ public struct FfiConverterTypeAuthUser: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthUser_lift(_ buf: RustBuffer) throws -> AuthUser { - return try FfiConverterTypeAuthUser.lift(buf) +public func FfiConverterTypeCoreConfig_lift(_ buf: RustBuffer) throws -> CoreConfig { + return try FfiConverterTypeCoreConfig.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeAuthUser_lower(_ value: AuthUser) -> RustBuffer { - return FfiConverterTypeAuthUser.lower(value) +public func FfiConverterTypeCoreConfig_lower(_ value: CoreConfig) -> RustBuffer { + return FfiConverterTypeCoreConfig.lower(value) } -public struct BoxPrim: Equatable, Hashable { - public var x: Float - public var y: Float - public var w: Float - public var h: Float - public var radius: Float - public var style: BoxStyle - public var color: ColorRole - public var scroller: UInt32? +/** + * A debug/demo transcript entry (markdown text), for fixtures and benches. + */ +public struct DebugEntry: Equatable, Hashable { + public var id: String + public var user: Bool + public var text: String + public var streaming: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(x: Float, y: Float, w: Float, h: Float, radius: Float, style: BoxStyle, color: ColorRole, scroller: UInt32?) { - self.x = x - self.y = y - self.w = w - self.h = h - self.radius = radius - self.style = style - self.color = color - self.scroller = scroller + public init(id: String, user: Bool, text: String, streaming: Bool) { + self.id = id + self.user = user + self.text = text + self.streaming = streaming } @@ -4992,36 +6195,28 @@ public struct BoxPrim: Equatable, Hashable { } #if compiler(>=6) -extension BoxPrim: Sendable {} -#endif - -#if swift(>=5.8) -@_documentation(visibility: private) +extension DebugEntry: Sendable {} #endif -public struct FfiConverterTypeBoxPrim: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> BoxPrim { - return - try BoxPrim( - x: FfiConverterFloat.read(from: &buf), - y: FfiConverterFloat.read(from: &buf), - w: FfiConverterFloat.read(from: &buf), - h: FfiConverterFloat.read(from: &buf), - radius: FfiConverterFloat.read(from: &buf), - style: FfiConverterTypeBoxStyle.read(from: &buf), - color: FfiConverterTypeColorRole.read(from: &buf), - scroller: FfiConverterOptionUInt32.read(from: &buf) + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeDebugEntry: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DebugEntry { + return + try DebugEntry( + id: FfiConverterString.read(from: &buf), + user: FfiConverterBool.read(from: &buf), + text: FfiConverterString.read(from: &buf), + streaming: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: BoxPrim, into buf: inout [UInt8]) { - FfiConverterFloat.write(value.x, into: &buf) - FfiConverterFloat.write(value.y, into: &buf) - FfiConverterFloat.write(value.w, into: &buf) - FfiConverterFloat.write(value.h, into: &buf) - FfiConverterFloat.write(value.radius, into: &buf) - FfiConverterTypeBoxStyle.write(value.style, into: &buf) - FfiConverterTypeColorRole.write(value.color, into: &buf) - FfiConverterOptionUInt32.write(value.scroller, into: &buf) + public static func write(_ value: DebugEntry, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterBool.write(value.user, into: &buf) + FfiConverterString.write(value.text, into: &buf) + FfiConverterBool.write(value.streaming, into: &buf) } } @@ -5029,51 +6224,43 @@ public struct FfiConverterTypeBoxPrim: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeBoxPrim_lift(_ buf: RustBuffer) throws -> BoxPrim { - return try FfiConverterTypeBoxPrim.lift(buf) +public func FfiConverterTypeDebugEntry_lift(_ buf: RustBuffer) throws -> DebugEntry { + return try FfiConverterTypeDebugEntry.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeBoxPrim_lower(_ value: BoxPrim) -> RustBuffer { - return FfiConverterTypeBoxPrim.lower(value) +public func FfiConverterTypeDebugEntry_lower(_ value: DebugEntry) -> RustBuffer { + return FfiConverterTypeDebugEntry.lower(value) } -public struct CaptionFrame: Equatable, Hashable { - /** - * The tail of the utterance to show (ellipsized in front when long). - */ - public var text: String - public var spans: [CaptionSpan] +public struct DemoOptions: Equatable, Hashable { + public var fixture: DemoFixture /** - * The previous utterance while it fades out, and its opacity. + * Size of the flagship chat's (`chat-veil`) transcript. */ - public var previous: String? - public var previousAlpha: Float + public var transcriptScale: TranscriptScale + public var streamSpeed: StreamSpeed /** - * Something is still fading: ask for the next frame. + * Repeat the scripted reply 12× (long streaming stress). */ - public var animating: Bool + public var longReply: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init( - /** - * The tail of the utterance to show (ellipsized in front when long). - */text: String, spans: [CaptionSpan], + public init(fixture: DemoFixture, /** - * The previous utterance while it fades out, and its opacity. - */previous: String?, previousAlpha: Float, + * Size of the flagship chat's (`chat-veil`) transcript. + */transcriptScale: TranscriptScale, streamSpeed: StreamSpeed, /** - * Something is still fading: ask for the next frame. - */animating: Bool) { - self.text = text - self.spans = spans - self.previous = previous - self.previousAlpha = previousAlpha - self.animating = animating + * Repeat the scripted reply 12× (long streaming stress). + */longReply: Bool) { + self.fixture = fixture + self.transcriptScale = transcriptScale + self.streamSpeed = streamSpeed + self.longReply = longReply } @@ -5082,30 +6269,28 @@ public struct CaptionFrame: Equatable, Hashable { } #if compiler(>=6) -extension CaptionFrame: Sendable {} +extension DemoOptions: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeCaptionFrame: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CaptionFrame { +public struct FfiConverterTypeDemoOptions: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DemoOptions { return - try CaptionFrame( - text: FfiConverterString.read(from: &buf), - spans: FfiConverterSequenceTypeCaptionSpan.read(from: &buf), - previous: FfiConverterOptionString.read(from: &buf), - previousAlpha: FfiConverterFloat.read(from: &buf), - animating: FfiConverterBool.read(from: &buf) + try DemoOptions( + fixture: FfiConverterTypeDemoFixture.read(from: &buf), + transcriptScale: FfiConverterTypeTranscriptScale.read(from: &buf), + streamSpeed: FfiConverterTypeStreamSpeed.read(from: &buf), + longReply: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: CaptionFrame, into buf: inout [UInt8]) { - FfiConverterString.write(value.text, into: &buf) - FfiConverterSequenceTypeCaptionSpan.write(value.spans, into: &buf) - FfiConverterOptionString.write(value.previous, into: &buf) - FfiConverterFloat.write(value.previousAlpha, into: &buf) - FfiConverterBool.write(value.animating, into: &buf) + public static func write(_ value: DemoOptions, into buf: inout [UInt8]) { + FfiConverterTypeDemoFixture.write(value.fixture, into: &buf) + FfiConverterTypeTranscriptScale.write(value.transcriptScale, into: &buf) + FfiConverterTypeStreamSpeed.write(value.streamSpeed, into: &buf) + FfiConverterBool.write(value.longReply, into: &buf) } } @@ -5113,32 +6298,49 @@ public struct FfiConverterTypeCaptionFrame: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeCaptionFrame_lift(_ buf: RustBuffer) throws -> CaptionFrame { - return try FfiConverterTypeCaptionFrame.lift(buf) +public func FfiConverterTypeDemoOptions_lift(_ buf: RustBuffer) throws -> DemoOptions { + return try FfiConverterTypeDemoOptions.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeCaptionFrame_lower(_ value: CaptionFrame) -> RustBuffer { - return FfiConverterTypeCaptionFrame.lower(value) +public func FfiConverterTypeDemoOptions_lower(_ value: DemoOptions) -> RustBuffer { + return FfiConverterTypeDemoOptions.lower(value) } -/** - * A veiled UTF-16 range of the caption and its opacity (0..1). - */ -public struct CaptionSpan: Equatable, Hashable { - public var start: UInt32 - public var end: UInt32 - public var alpha: Float +public struct DeviceView: Equatable, Hashable { + public var id: String + public var name: String + public var platform: String + public var online: Bool + public var lastSeenMs: Int64? + public var version: String? + public var capabilities: [String] + /** + * Can run sessions (desktop/server engines — not phones). + */ + public var isExecutionHost: Bool + public var isSelf: Bool + public var sessionCount: UInt32 // Default memberwise initializers are never public by default, so we // declare one manually. - public init(start: UInt32, end: UInt32, alpha: Float) { - self.start = start - self.end = end - self.alpha = alpha + public init(id: String, name: String, platform: String, online: Bool, lastSeenMs: Int64?, version: String?, capabilities: [String], + /** + * Can run sessions (desktop/server engines — not phones). + */isExecutionHost: Bool, isSelf: Bool, sessionCount: UInt32) { + self.id = id + self.name = name + self.platform = platform + self.online = online + self.lastSeenMs = lastSeenMs + self.version = version + self.capabilities = capabilities + self.isExecutionHost = isExecutionHost + self.isSelf = isSelf + self.sessionCount = sessionCount } @@ -5147,26 +6349,40 @@ public struct CaptionSpan: Equatable, Hashable { } #if compiler(>=6) -extension CaptionSpan: Sendable {} +extension DeviceView: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeCaptionSpan: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CaptionSpan { +public struct FfiConverterTypeDeviceView: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DeviceView { return - try CaptionSpan( - start: FfiConverterUInt32.read(from: &buf), - end: FfiConverterUInt32.read(from: &buf), - alpha: FfiConverterFloat.read(from: &buf) + try DeviceView( + id: FfiConverterString.read(from: &buf), + name: FfiConverterString.read(from: &buf), + platform: FfiConverterString.read(from: &buf), + online: FfiConverterBool.read(from: &buf), + lastSeenMs: FfiConverterOptionInt64.read(from: &buf), + version: FfiConverterOptionString.read(from: &buf), + capabilities: FfiConverterSequenceString.read(from: &buf), + isExecutionHost: FfiConverterBool.read(from: &buf), + isSelf: FfiConverterBool.read(from: &buf), + sessionCount: FfiConverterUInt32.read(from: &buf) ) } - public static func write(_ value: CaptionSpan, into buf: inout [UInt8]) { - FfiConverterUInt32.write(value.start, into: &buf) - FfiConverterUInt32.write(value.end, into: &buf) - FfiConverterFloat.write(value.alpha, into: &buf) + public static func write(_ value: DeviceView, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterString.write(value.name, into: &buf) + FfiConverterString.write(value.platform, into: &buf) + FfiConverterBool.write(value.online, into: &buf) + FfiConverterOptionInt64.write(value.lastSeenMs, into: &buf) + FfiConverterOptionString.write(value.version, into: &buf) + FfiConverterSequenceString.write(value.capabilities, into: &buf) + FfiConverterBool.write(value.isExecutionHost, into: &buf) + FfiConverterBool.write(value.isSelf, into: &buf) + FfiConverterUInt32.write(value.sessionCount, into: &buf) } } @@ -5174,54 +6390,27 @@ public struct FfiConverterTypeCaptionSpan: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeCaptionSpan_lift(_ buf: RustBuffer) throws -> CaptionSpan { - return try FfiConverterTypeCaptionSpan.lift(buf) +public func FfiConverterTypeDeviceView_lift(_ buf: RustBuffer) throws -> DeviceView { + return try FfiConverterTypeDeviceView.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeCaptionSpan_lower(_ value: CaptionSpan) -> RustBuffer { - return FfiConverterTypeCaptionSpan.lower(value) +public func FfiConverterTypeDeviceView_lower(_ value: DeviceView) -> RustBuffer { + return FfiConverterTypeDeviceView.lower(value) } -/** - * A chat's run configuration. Ids are wire strings. - */ -public struct ChatConfig: Equatable, Hashable { - /** - * `claude-code`, `codex`, … - */ - public var harness: String - public var model: String? - /** - * `low` … `ultrathink`. - */ - public var reasoning: String? - /** - * Model option id → choice id (`contextWindow` → `1m`). - */ - public var modelOptions: [String: String] - public var sandbox: SandboxLevel +public struct FaceData: Equatable, Hashable { + public var role: FaceRole + public var bytes: Data // Default memberwise initializers are never public by default, so we // declare one manually. - public init( - /** - * `claude-code`, `codex`, … - */harness: String, model: String?, - /** - * `low` … `ultrathink`. - */reasoning: String?, - /** - * Model option id → choice id (`contextWindow` → `1m`). - */modelOptions: [String: String], sandbox: SandboxLevel) { - self.harness = harness - self.model = model - self.reasoning = reasoning - self.modelOptions = modelOptions - self.sandbox = sandbox + public init(role: FaceRole, bytes: Data) { + self.role = role + self.bytes = bytes } @@ -5230,30 +6419,24 @@ public struct ChatConfig: Equatable, Hashable { } #if compiler(>=6) -extension ChatConfig: Sendable {} +extension FaceData: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeChatConfig: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatConfig { +public struct FfiConverterTypeFaceData: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FaceData { return - try ChatConfig( - harness: FfiConverterString.read(from: &buf), - model: FfiConverterOptionString.read(from: &buf), - reasoning: FfiConverterOptionString.read(from: &buf), - modelOptions: FfiConverterDictionaryStringString.read(from: &buf), - sandbox: FfiConverterTypeSandboxLevel.read(from: &buf) + try FaceData( + role: FfiConverterTypeFaceRole.read(from: &buf), + bytes: FfiConverterData.read(from: &buf) ) } - public static func write(_ value: ChatConfig, into buf: inout [UInt8]) { - FfiConverterString.write(value.harness, into: &buf) - FfiConverterOptionString.write(value.model, into: &buf) - FfiConverterOptionString.write(value.reasoning, into: &buf) - FfiConverterDictionaryStringString.write(value.modelOptions, into: &buf) - FfiConverterTypeSandboxLevel.write(value.sandbox, into: &buf) + public static func write(_ value: FaceData, into buf: inout [UInt8]) { + FfiConverterTypeFaceRole.write(value.role, into: &buf) + FfiConverterData.write(value.bytes, into: &buf) } } @@ -5261,104 +6444,40 @@ public struct FfiConverterTypeChatConfig: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeChatConfig_lift(_ buf: RustBuffer) throws -> ChatConfig { - return try FfiConverterTypeChatConfig.lift(buf) +public func FfiConverterTypeFaceData_lift(_ buf: RustBuffer) throws -> FaceData { + return try FfiConverterTypeFaceData.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeChatConfig_lower(_ value: ChatConfig) -> RustBuffer { - return FfiConverterTypeChatConfig.lower(value) +public func FfiConverterTypeFaceData_lower(_ value: FaceData) -> RustBuffer { + return FfiConverterTypeFaceData.lower(value) } /** - * Everything the composer and its status strip render. + * An overflow fade: runs whose baseline falls inside the rect's vertical + * band are painted through an alpha ramp — the text itself fades, so it + * works over any background. */ -public struct ComposerState: Equatable, Hashable { - public var chatId: String - public var revision: UInt64 - public var title: String - public var host: HostInfo - public var live: LiveStatus - /** - * The shared queue, delivery order. - */ - public var queue: [QueueItem] - /** - * This device's unadopted sends, oldest first. - */ - public var pendingSends: [PendingSend] - /** - * Headline state of the oldest pending send. - */ - public var sendState: SendState? - /** - * A send now would queue rather than deliver promptly. - */ - public var deliveryDegraded: Bool - public var room: RoomState - /** - * Attachment escort progress (0..1). - */ - public var transferProgress: Double? - /** - * Answer this instead of typing. - */ - public var openInput: InputRequest? - /** - * Last queue action failure. - */ - public var queueError: String? - /** - * Scroll-to-own-send target. - */ - public var lastSubmittedMessageId: String? - public var contextUsage: ContextUsage? +public struct Fade: Equatable, Hashable { + public var x: Float + public var y: Float + public var w: Float + public var h: Float + public var edge: FadeEdge + public var scroller: UInt32? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(chatId: String, revision: UInt64, title: String, host: HostInfo, live: LiveStatus, - /** - * The shared queue, delivery order. - */queue: [QueueItem], - /** - * This device's unadopted sends, oldest first. - */pendingSends: [PendingSend], - /** - * Headline state of the oldest pending send. - */sendState: SendState?, - /** - * A send now would queue rather than deliver promptly. - */deliveryDegraded: Bool, room: RoomState, - /** - * Attachment escort progress (0..1). - */transferProgress: Double?, - /** - * Answer this instead of typing. - */openInput: InputRequest?, - /** - * Last queue action failure. - */queueError: String?, - /** - * Scroll-to-own-send target. - */lastSubmittedMessageId: String?, contextUsage: ContextUsage?) { - self.chatId = chatId - self.revision = revision - self.title = title - self.host = host - self.live = live - self.queue = queue - self.pendingSends = pendingSends - self.sendState = sendState - self.deliveryDegraded = deliveryDegraded - self.room = room - self.transferProgress = transferProgress - self.openInput = openInput - self.queueError = queueError - self.lastSubmittedMessageId = lastSubmittedMessageId - self.contextUsage = contextUsage + public init(x: Float, y: Float, w: Float, h: Float, edge: FadeEdge, scroller: UInt32?) { + self.x = x + self.y = y + self.w = w + self.h = h + self.edge = edge + self.scroller = scroller } @@ -5367,50 +6486,32 @@ public struct ComposerState: Equatable, Hashable { } #if compiler(>=6) -extension ComposerState: Sendable {} +extension Fade: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeComposerState: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ComposerState { +public struct FfiConverterTypeFade: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Fade { return - try ComposerState( - chatId: FfiConverterString.read(from: &buf), - revision: FfiConverterUInt64.read(from: &buf), - title: FfiConverterString.read(from: &buf), - host: FfiConverterTypeHostInfo.read(from: &buf), - live: FfiConverterTypeLiveStatus.read(from: &buf), - queue: FfiConverterSequenceTypeQueueItem.read(from: &buf), - pendingSends: FfiConverterSequenceTypePendingSend.read(from: &buf), - sendState: FfiConverterOptionTypeSendState.read(from: &buf), - deliveryDegraded: FfiConverterBool.read(from: &buf), - room: FfiConverterTypeRoomState.read(from: &buf), - transferProgress: FfiConverterOptionDouble.read(from: &buf), - openInput: FfiConverterOptionTypeInputRequest.read(from: &buf), - queueError: FfiConverterOptionString.read(from: &buf), - lastSubmittedMessageId: FfiConverterOptionString.read(from: &buf), - contextUsage: FfiConverterOptionTypeContextUsage.read(from: &buf) + try Fade( + x: FfiConverterFloat.read(from: &buf), + y: FfiConverterFloat.read(from: &buf), + w: FfiConverterFloat.read(from: &buf), + h: FfiConverterFloat.read(from: &buf), + edge: FfiConverterTypeFadeEdge.read(from: &buf), + scroller: FfiConverterOptionUInt32.read(from: &buf) ) } - public static func write(_ value: ComposerState, into buf: inout [UInt8]) { - FfiConverterString.write(value.chatId, into: &buf) - FfiConverterUInt64.write(value.revision, into: &buf) - FfiConverterString.write(value.title, into: &buf) - FfiConverterTypeHostInfo.write(value.host, into: &buf) - FfiConverterTypeLiveStatus.write(value.live, into: &buf) - FfiConverterSequenceTypeQueueItem.write(value.queue, into: &buf) - FfiConverterSequenceTypePendingSend.write(value.pendingSends, into: &buf) - FfiConverterOptionTypeSendState.write(value.sendState, into: &buf) - FfiConverterBool.write(value.deliveryDegraded, into: &buf) - FfiConverterTypeRoomState.write(value.room, into: &buf) - FfiConverterOptionDouble.write(value.transferProgress, into: &buf) - FfiConverterOptionTypeInputRequest.write(value.openInput, into: &buf) - FfiConverterOptionString.write(value.queueError, into: &buf) - FfiConverterOptionString.write(value.lastSubmittedMessageId, into: &buf) - FfiConverterOptionTypeContextUsage.write(value.contextUsage, into: &buf) + public static func write(_ value: Fade, into buf: inout [UInt8]) { + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.w, into: &buf) + FfiConverterFloat.write(value.h, into: &buf) + FfiConverterTypeFadeEdge.write(value.edge, into: &buf) + FfiConverterOptionUInt32.write(value.scroller, into: &buf) } } @@ -5418,49 +6519,30 @@ public struct FfiConverterTypeComposerState: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeComposerState_lift(_ buf: RustBuffer) throws -> ComposerState { - return try FfiConverterTypeComposerState.lift(buf) +public func FfiConverterTypeFade_lift(_ buf: RustBuffer) throws -> Fade { + return try FfiConverterTypeFade.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeComposerState_lower(_ value: ComposerState) -> RustBuffer { - return FfiConverterTypeComposerState.lower(value) +public func FfiConverterTypeFade_lower(_ value: Fade) -> RustBuffer { + return FfiConverterTypeFade.lower(value) } -public struct Connectivity: Equatable, Hashable { - public var state: ConnectivityState - /** - * Epoch ms of the next scheduled redial (countdown), if any. - */ - public var retryAtMs: Int64? - /** - * The failure that started the current outage. - */ - public var lastFailure: String? - /** - * Open chats whose room is graced-degraded. - */ - public var degradedChats: [String] +/** + * A workspace file or folder for `@` mentions. + */ +public struct FileMatch: Equatable, Hashable { + public var path: String + public var isDir: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(state: ConnectivityState, - /** - * Epoch ms of the next scheduled redial (countdown), if any. - */retryAtMs: Int64?, - /** - * The failure that started the current outage. - */lastFailure: String?, - /** - * Open chats whose room is graced-degraded. - */degradedChats: [String]) { - self.state = state - self.retryAtMs = retryAtMs - self.lastFailure = lastFailure - self.degradedChats = degradedChats + public init(path: String, isDir: Bool) { + self.path = path + self.isDir = isDir } @@ -5469,28 +6551,24 @@ public struct Connectivity: Equatable, Hashable { } #if compiler(>=6) -extension Connectivity: Sendable {} +extension FileMatch: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeConnectivity: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Connectivity { +public struct FfiConverterTypeFileMatch: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FileMatch { return - try Connectivity( - state: FfiConverterTypeConnectivityState.read(from: &buf), - retryAtMs: FfiConverterOptionInt64.read(from: &buf), - lastFailure: FfiConverterOptionString.read(from: &buf), - degradedChats: FfiConverterSequenceString.read(from: &buf) + try FileMatch( + path: FfiConverterString.read(from: &buf), + isDir: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: Connectivity, into buf: inout [UInt8]) { - FfiConverterTypeConnectivityState.write(value.state, into: &buf) - FfiConverterOptionInt64.write(value.retryAtMs, into: &buf) - FfiConverterOptionString.write(value.lastFailure, into: &buf) - FfiConverterSequenceString.write(value.degradedChats, into: &buf) + public static func write(_ value: FileMatch, into buf: inout [UInt8]) { + FfiConverterString.write(value.path, into: &buf) + FfiConverterBool.write(value.isDir, into: &buf) } } @@ -5498,27 +6576,29 @@ public struct FfiConverterTypeConnectivity: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeConnectivity_lift(_ buf: RustBuffer) throws -> Connectivity { - return try FfiConverterTypeConnectivity.lift(buf) +public func FfiConverterTypeFileMatch_lift(_ buf: RustBuffer) throws -> FileMatch { + return try FfiConverterTypeFileMatch.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeConnectivity_lower(_ value: Connectivity) -> RustBuffer { - return FfiConverterTypeConnectivity.lower(value) +public func FfiConverterTypeFileMatch_lower(_ value: FileMatch) -> RustBuffer { + return FfiConverterTypeFileMatch.lower(value) } -public struct ContextUsage: Equatable, Hashable { - public var tokens: UInt64? - public var window: UInt64? +public struct FolderEntry: Equatable, Hashable { + public var name: String + public var isDir: Bool + public var isRepo: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(tokens: UInt64?, window: UInt64?) { - self.tokens = tokens - self.window = window + public init(name: String, isDir: Bool, isRepo: Bool) { + self.name = name + self.isDir = isDir + self.isRepo = isRepo } @@ -5527,24 +6607,26 @@ public struct ContextUsage: Equatable, Hashable { } #if compiler(>=6) -extension ContextUsage: Sendable {} +extension FolderEntry: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeContextUsage: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ContextUsage { +public struct FfiConverterTypeFolderEntry: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FolderEntry { return - try ContextUsage( - tokens: FfiConverterOptionUInt64.read(from: &buf), - window: FfiConverterOptionUInt64.read(from: &buf) + try FolderEntry( + name: FfiConverterString.read(from: &buf), + isDir: FfiConverterBool.read(from: &buf), + isRepo: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: ContextUsage, into buf: inout [UInt8]) { - FfiConverterOptionUInt64.write(value.tokens, into: &buf) - FfiConverterOptionUInt64.write(value.window, into: &buf) + public static func write(_ value: FolderEntry, into buf: inout [UInt8]) { + FfiConverterString.write(value.name, into: &buf) + FfiConverterBool.write(value.isDir, into: &buf) + FfiConverterBool.write(value.isRepo, into: &buf) } } @@ -5552,70 +6634,29 @@ public struct FfiConverterTypeContextUsage: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeContextUsage_lift(_ buf: RustBuffer) throws -> ContextUsage { - return try FfiConverterTypeContextUsage.lift(buf) +public func FfiConverterTypeFolderEntry_lift(_ buf: RustBuffer) throws -> FolderEntry { + return try FfiConverterTypeFolderEntry.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeContextUsage_lower(_ value: ContextUsage) -> RustBuffer { - return FfiConverterTypeContextUsage.lower(value) +public func FfiConverterTypeFolderEntry_lower(_ value: FolderEntry) -> RustBuffer { + return FfiConverterTypeFolderEntry.lower(value) } -/** - * Static per-install configuration. - */ -public struct CoreConfig: Equatable, Hashable { - /** - * Edge base URL (`auth_production_edge_url()` for prod). - */ - public var edgeUrl: String - /** - * Writable directory for registry/doc snapshots and caches, scoped by - * the platform to the signed-in identity (sign-out wipes it). - */ - public var dataDir: String - /** - * Stable device id (`ios-xxxxxxxx`), stamped on every command/queue row. - */ - public var deviceId: String - /** - * Human name for presence/attribution ("Wing's iPhone"). - */ - public var deviceName: String - /** - * `ios` / `android`. - */ - public var platform: String - public var appVersion: String +public struct FolderListing: Equatable, Hashable { + public var path: String + public var entries: [FolderEntry] + public var truncated: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init( - /** - * Edge base URL (`auth_production_edge_url()` for prod). - */edgeUrl: String, - /** - * Writable directory for registry/doc snapshots and caches, scoped by - * the platform to the signed-in identity (sign-out wipes it). - */dataDir: String, - /** - * Stable device id (`ios-xxxxxxxx`), stamped on every command/queue row. - */deviceId: String, - /** - * Human name for presence/attribution ("Wing's iPhone"). - */deviceName: String, - /** - * `ios` / `android`. - */platform: String, appVersion: String) { - self.edgeUrl = edgeUrl - self.dataDir = dataDir - self.deviceId = deviceId - self.deviceName = deviceName - self.platform = platform - self.appVersion = appVersion + public init(path: String, entries: [FolderEntry], truncated: Bool) { + self.path = path + self.entries = entries + self.truncated = truncated } @@ -5624,32 +6665,26 @@ public struct CoreConfig: Equatable, Hashable { } #if compiler(>=6) -extension CoreConfig: Sendable {} +extension FolderListing: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeCoreConfig: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CoreConfig { +public struct FfiConverterTypeFolderListing: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FolderListing { return - try CoreConfig( - edgeUrl: FfiConverterString.read(from: &buf), - dataDir: FfiConverterString.read(from: &buf), - deviceId: FfiConverterString.read(from: &buf), - deviceName: FfiConverterString.read(from: &buf), - platform: FfiConverterString.read(from: &buf), - appVersion: FfiConverterString.read(from: &buf) + try FolderListing( + path: FfiConverterString.read(from: &buf), + entries: FfiConverterSequenceTypeFolderEntry.read(from: &buf), + truncated: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: CoreConfig, into buf: inout [UInt8]) { - FfiConverterString.write(value.edgeUrl, into: &buf) - FfiConverterString.write(value.dataDir, into: &buf) - FfiConverterString.write(value.deviceId, into: &buf) - FfiConverterString.write(value.deviceName, into: &buf) - FfiConverterString.write(value.platform, into: &buf) - FfiConverterString.write(value.appVersion, into: &buf) + public static func write(_ value: FolderListing, into buf: inout [UInt8]) { + FfiConverterString.write(value.path, into: &buf) + FfiConverterSequenceTypeFolderEntry.write(value.entries, into: &buf) + FfiConverterBool.write(value.truncated, into: &buf) } } @@ -5657,34 +6692,33 @@ public struct FfiConverterTypeCoreConfig: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeCoreConfig_lift(_ buf: RustBuffer) throws -> CoreConfig { - return try FfiConverterTypeCoreConfig.lift(buf) +public func FfiConverterTypeFolderListing_lift(_ buf: RustBuffer) throws -> FolderListing { + return try FfiConverterTypeFolderListing.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeCoreConfig_lower(_ value: CoreConfig) -> RustBuffer { - return FfiConverterTypeCoreConfig.lower(value) +public func FfiConverterTypeFolderListing_lower(_ value: FolderListing) -> RustBuffer { + return FfiConverterTypeFolderListing.lower(value) } /** - * A debug/demo transcript entry (markdown text), for fixtures and benches. + * Premultiplied BGRA8, top row first (CoreGraphics' native + * `byteOrder32Little | premultipliedFirst`). */ -public struct DebugEntry: Equatable, Hashable { - public var id: String - public var user: Bool - public var text: String - public var streaming: Bool +public struct FrameImage: Equatable, Hashable { + public var width: UInt32 + public var height: UInt32 + public var bgra: Data // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, user: Bool, text: String, streaming: Bool) { - self.id = id - self.user = user - self.text = text - self.streaming = streaming + public init(width: UInt32, height: UInt32, bgra: Data) { + self.width = width + self.height = height + self.bgra = bgra } @@ -5693,28 +6727,26 @@ public struct DebugEntry: Equatable, Hashable { } #if compiler(>=6) -extension DebugEntry: Sendable {} +extension FrameImage: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeDebugEntry: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DebugEntry { +public struct FfiConverterTypeFrameImage: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FrameImage { return - try DebugEntry( - id: FfiConverterString.read(from: &buf), - user: FfiConverterBool.read(from: &buf), - text: FfiConverterString.read(from: &buf), - streaming: FfiConverterBool.read(from: &buf) + try FrameImage( + width: FfiConverterUInt32.read(from: &buf), + height: FfiConverterUInt32.read(from: &buf), + bgra: FfiConverterData.read(from: &buf) ) } - public static func write(_ value: DebugEntry, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterBool.write(value.user, into: &buf) - FfiConverterString.write(value.text, into: &buf) - FfiConverterBool.write(value.streaming, into: &buf) + public static func write(_ value: FrameImage, into buf: inout [UInt8]) { + FfiConverterUInt32.write(value.width, into: &buf) + FfiConverterUInt32.write(value.height, into: &buf) + FfiConverterData.write(value.bgra, into: &buf) } } @@ -5722,43 +6754,50 @@ public struct FfiConverterTypeDebugEntry: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeDebugEntry_lift(_ buf: RustBuffer) throws -> DebugEntry { - return try FfiConverterTypeDebugEntry.lift(buf) +public func FfiConverterTypeFrameImage_lift(_ buf: RustBuffer) throws -> FrameImage { + return try FfiConverterTypeFrameImage.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeDebugEntry_lower(_ value: DebugEntry) -> RustBuffer { - return FfiConverterTypeDebugEntry.lower(value) +public func FfiConverterTypeFrameImage_lower(_ value: FrameImage) -> RustBuffer { + return FfiConverterTypeFrameImage.lower(value) } -public struct DemoOptions: Equatable, Hashable { - public var fixture: DemoFixture +/** + * The Threads page — the desktop sidebar order. + */ +public struct FrontPage: Equatable, Hashable { /** - * Size of the flagship chat's (`chat-veil`) transcript. + * Shared manual pin order. */ - public var transcriptScale: TranscriptScale - public var streamSpeed: StreamSpeed + public var pinned: [SessionRow] /** - * Repeat the scripted reply 12× (long streaming stress). + * User sections ("folders"), each in recency order. */ - public var longReply: Bool + public var sections: [SectionView] + /** + * Everything else, by recency. + */ + public var recent: [SessionRow] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(fixture: DemoFixture, + public init( /** - * Size of the flagship chat's (`chat-veil`) transcript. - */transcriptScale: TranscriptScale, streamSpeed: StreamSpeed, + * Shared manual pin order. + */pinned: [SessionRow], /** - * Repeat the scripted reply 12× (long streaming stress). - */longReply: Bool) { - self.fixture = fixture - self.transcriptScale = transcriptScale - self.streamSpeed = streamSpeed - self.longReply = longReply + * User sections ("folders"), each in recency order. + */sections: [SectionView], + /** + * Everything else, by recency. + */recent: [SessionRow]) { + self.pinned = pinned + self.sections = sections + self.recent = recent } @@ -5767,28 +6806,26 @@ public struct DemoOptions: Equatable, Hashable { } #if compiler(>=6) -extension DemoOptions: Sendable {} +extension FrontPage: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeDemoOptions: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DemoOptions { +public struct FfiConverterTypeFrontPage: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FrontPage { return - try DemoOptions( - fixture: FfiConverterTypeDemoFixture.read(from: &buf), - transcriptScale: FfiConverterTypeTranscriptScale.read(from: &buf), - streamSpeed: FfiConverterTypeStreamSpeed.read(from: &buf), - longReply: FfiConverterBool.read(from: &buf) + try FrontPage( + pinned: FfiConverterSequenceTypeSessionRow.read(from: &buf), + sections: FfiConverterSequenceTypeSectionView.read(from: &buf), + recent: FfiConverterSequenceTypeSessionRow.read(from: &buf) ) } - public static func write(_ value: DemoOptions, into buf: inout [UInt8]) { - FfiConverterTypeDemoFixture.write(value.fixture, into: &buf) - FfiConverterTypeTranscriptScale.write(value.transcriptScale, into: &buf) - FfiConverterTypeStreamSpeed.write(value.streamSpeed, into: &buf) - FfiConverterBool.write(value.longReply, into: &buf) + public static func write(_ value: FrontPage, into buf: inout [UInt8]) { + FfiConverterSequenceTypeSessionRow.write(value.pinned, into: &buf) + FfiConverterSequenceTypeSectionView.write(value.sections, into: &buf) + FfiConverterSequenceTypeSessionRow.write(value.recent, into: &buf) } } @@ -5796,49 +6833,59 @@ public struct FfiConverterTypeDemoOptions: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeDemoOptions_lift(_ buf: RustBuffer) throws -> DemoOptions { - return try FfiConverterTypeDemoOptions.lift(buf) +public func FfiConverterTypeFrontPage_lift(_ buf: RustBuffer) throws -> FrontPage { + return try FfiConverterTypeFrontPage.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeDemoOptions_lower(_ value: DemoOptions) -> RustBuffer { - return FfiConverterTypeDemoOptions.lower(value) +public func FfiConverterTypeFrontPage_lower(_ value: FrontPage) -> RustBuffer { + return FfiConverterTypeFrontPage.lower(value) } -public struct DeviceView: Equatable, Hashable { +public struct HarnessInfo: Equatable, Hashable { public var id: String - public var name: String - public var platform: String - public var online: Bool - public var lastSeenMs: Int64? - public var version: String? - public var capabilities: [String] + public var label: String + public var supportsSteering: Bool? /** - * Can run sessions (desktop/server engines — not phones). + * `step-boundary` / `turn-boundary`. */ - public var isExecutionHost: Bool - public var isSelf: Bool - public var sessionCount: UInt32 + public var steeringMode: String? + public var reasoningLevels: [String] + public var installed: Bool + public var enabled: Bool? + /** + * Installed and not disabled on the device. + */ + public var offered: Bool + /** + * Steers mid-turn (unknown until a live catalog). + */ + public var midTurnSteering: Bool? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, name: String, platform: String, online: Bool, lastSeenMs: Int64?, version: String?, capabilities: [String], + public init(id: String, label: String, supportsSteering: Bool?, /** - * Can run sessions (desktop/server engines — not phones). - */isExecutionHost: Bool, isSelf: Bool, sessionCount: UInt32) { - self.id = id - self.name = name - self.platform = platform - self.online = online - self.lastSeenMs = lastSeenMs - self.version = version - self.capabilities = capabilities - self.isExecutionHost = isExecutionHost - self.isSelf = isSelf - self.sessionCount = sessionCount + * `step-boundary` / `turn-boundary`. + */steeringMode: String?, reasoningLevels: [String], installed: Bool, enabled: Bool?, + /** + * Installed and not disabled on the device. + */offered: Bool, + /** + * Steers mid-turn (unknown until a live catalog). + */midTurnSteering: Bool?) { + self.id = id + self.label = label + self.supportsSteering = supportsSteering + self.steeringMode = steeringMode + self.reasoningLevels = reasoningLevels + self.installed = installed + self.enabled = enabled + self.offered = offered + self.midTurnSteering = midTurnSteering } @@ -5847,40 +6894,38 @@ public struct DeviceView: Equatable, Hashable { } #if compiler(>=6) -extension DeviceView: Sendable {} +extension HarnessInfo: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeDeviceView: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DeviceView { +public struct FfiConverterTypeHarnessInfo: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HarnessInfo { return - try DeviceView( + try HarnessInfo( id: FfiConverterString.read(from: &buf), - name: FfiConverterString.read(from: &buf), - platform: FfiConverterString.read(from: &buf), - online: FfiConverterBool.read(from: &buf), - lastSeenMs: FfiConverterOptionInt64.read(from: &buf), - version: FfiConverterOptionString.read(from: &buf), - capabilities: FfiConverterSequenceString.read(from: &buf), - isExecutionHost: FfiConverterBool.read(from: &buf), - isSelf: FfiConverterBool.read(from: &buf), - sessionCount: FfiConverterUInt32.read(from: &buf) + label: FfiConverterString.read(from: &buf), + supportsSteering: FfiConverterOptionBool.read(from: &buf), + steeringMode: FfiConverterOptionString.read(from: &buf), + reasoningLevels: FfiConverterSequenceString.read(from: &buf), + installed: FfiConverterBool.read(from: &buf), + enabled: FfiConverterOptionBool.read(from: &buf), + offered: FfiConverterBool.read(from: &buf), + midTurnSteering: FfiConverterOptionBool.read(from: &buf) ) } - public static func write(_ value: DeviceView, into buf: inout [UInt8]) { + public static func write(_ value: HarnessInfo, into buf: inout [UInt8]) { FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.name, into: &buf) - FfiConverterString.write(value.platform, into: &buf) - FfiConverterBool.write(value.online, into: &buf) - FfiConverterOptionInt64.write(value.lastSeenMs, into: &buf) - FfiConverterOptionString.write(value.version, into: &buf) - FfiConverterSequenceString.write(value.capabilities, into: &buf) - FfiConverterBool.write(value.isExecutionHost, into: &buf) - FfiConverterBool.write(value.isSelf, into: &buf) - FfiConverterUInt32.write(value.sessionCount, into: &buf) + FfiConverterString.write(value.label, into: &buf) + FfiConverterOptionBool.write(value.supportsSteering, into: &buf) + FfiConverterOptionString.write(value.steeringMode, into: &buf) + FfiConverterSequenceString.write(value.reasoningLevels, into: &buf) + FfiConverterBool.write(value.installed, into: &buf) + FfiConverterOptionBool.write(value.enabled, into: &buf) + FfiConverterBool.write(value.offered, into: &buf) + FfiConverterOptionBool.write(value.midTurnSteering, into: &buf) } } @@ -5888,27 +6933,49 @@ public struct FfiConverterTypeDeviceView: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeDeviceView_lift(_ buf: RustBuffer) throws -> DeviceView { - return try FfiConverterTypeDeviceView.lift(buf) +public func FfiConverterTypeHarnessInfo_lift(_ buf: RustBuffer) throws -> HarnessInfo { + return try FfiConverterTypeHarnessInfo.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeDeviceView_lower(_ value: DeviceView) -> RustBuffer { - return FfiConverterTypeDeviceView.lower(value) +public func FfiConverterTypeHarnessInfo_lower(_ value: HarnessInfo) -> RustBuffer { + return FfiConverterTypeHarnessInfo.lower(value) } -public struct FaceData: Equatable, Hashable { - public var role: FaceRole - public var bytes: Data +public struct HostCapabilities: Equatable, Hashable { + public var messageQueue: Bool + public var queueActions: Bool + public var queueAttachments: Bool + public var cleanAttachmentText: Bool + public var queueEditLease: Bool + /** + * Host ≥ 0.2.12: attachments ride `pending://` refs. + */ + public var queuedAttachments: Bool + /** + * The chat's harness steers mid-turn (unknown until a live catalog). + */ + public var midTurnSteering: Bool? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(role: FaceRole, bytes: Data) { - self.role = role - self.bytes = bytes + public init(messageQueue: Bool, queueActions: Bool, queueAttachments: Bool, cleanAttachmentText: Bool, queueEditLease: Bool, + /** + * Host ≥ 0.2.12: attachments ride `pending://` refs. + */queuedAttachments: Bool, + /** + * The chat's harness steers mid-turn (unknown until a live catalog). + */midTurnSteering: Bool?) { + self.messageQueue = messageQueue + self.queueActions = queueActions + self.queueAttachments = queueAttachments + self.cleanAttachmentText = cleanAttachmentText + self.queueEditLease = queueEditLease + self.queuedAttachments = queuedAttachments + self.midTurnSteering = midTurnSteering } @@ -5917,24 +6984,34 @@ public struct FaceData: Equatable, Hashable { } #if compiler(>=6) -extension FaceData: Sendable {} +extension HostCapabilities: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeFaceData: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FaceData { +public struct FfiConverterTypeHostCapabilities: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HostCapabilities { return - try FaceData( - role: FfiConverterTypeFaceRole.read(from: &buf), - bytes: FfiConverterData.read(from: &buf) + try HostCapabilities( + messageQueue: FfiConverterBool.read(from: &buf), + queueActions: FfiConverterBool.read(from: &buf), + queueAttachments: FfiConverterBool.read(from: &buf), + cleanAttachmentText: FfiConverterBool.read(from: &buf), + queueEditLease: FfiConverterBool.read(from: &buf), + queuedAttachments: FfiConverterBool.read(from: &buf), + midTurnSteering: FfiConverterOptionBool.read(from: &buf) ) } - public static func write(_ value: FaceData, into buf: inout [UInt8]) { - FfiConverterTypeFaceRole.write(value.role, into: &buf) - FfiConverterData.write(value.bytes, into: &buf) + public static func write(_ value: HostCapabilities, into buf: inout [UInt8]) { + FfiConverterBool.write(value.messageQueue, into: &buf) + FfiConverterBool.write(value.queueActions, into: &buf) + FfiConverterBool.write(value.queueAttachments, into: &buf) + FfiConverterBool.write(value.cleanAttachmentText, into: &buf) + FfiConverterBool.write(value.queueEditLease, into: &buf) + FfiConverterBool.write(value.queuedAttachments, into: &buf) + FfiConverterOptionBool.write(value.midTurnSteering, into: &buf) } } @@ -5942,40 +7019,31 @@ public struct FfiConverterTypeFaceData: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFaceData_lift(_ buf: RustBuffer) throws -> FaceData { - return try FfiConverterTypeFaceData.lift(buf) +public func FfiConverterTypeHostCapabilities_lift(_ buf: RustBuffer) throws -> HostCapabilities { + return try FfiConverterTypeHostCapabilities.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFaceData_lower(_ value: FaceData) -> RustBuffer { - return FfiConverterTypeFaceData.lower(value) +public func FfiConverterTypeHostCapabilities_lower(_ value: HostCapabilities) -> RustBuffer { + return FfiConverterTypeHostCapabilities.lower(value) } -/** - * An overflow fade: runs whose baseline falls inside the rect's vertical - * band are painted through an alpha ramp — the text itself fades, so it - * works over any background. - */ -public struct Fade: Equatable, Hashable { - public var x: Float - public var y: Float - public var w: Float - public var h: Float - public var edge: FadeEdge - public var scroller: UInt32? +public struct HostInfo: Equatable, Hashable { + public var deviceId: String + public var name: String? + public var online: Bool + public var capabilities: HostCapabilities // Default memberwise initializers are never public by default, so we // declare one manually. - public init(x: Float, y: Float, w: Float, h: Float, edge: FadeEdge, scroller: UInt32?) { - self.x = x - self.y = y - self.w = w - self.h = h - self.edge = edge - self.scroller = scroller + public init(deviceId: String, name: String?, online: Bool, capabilities: HostCapabilities) { + self.deviceId = deviceId + self.name = name + self.online = online + self.capabilities = capabilities } @@ -5984,32 +7052,28 @@ public struct Fade: Equatable, Hashable { } #if compiler(>=6) -extension Fade: Sendable {} +extension HostInfo: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeFade: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Fade { +public struct FfiConverterTypeHostInfo: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HostInfo { return - try Fade( - x: FfiConverterFloat.read(from: &buf), - y: FfiConverterFloat.read(from: &buf), - w: FfiConverterFloat.read(from: &buf), - h: FfiConverterFloat.read(from: &buf), - edge: FfiConverterTypeFadeEdge.read(from: &buf), - scroller: FfiConverterOptionUInt32.read(from: &buf) + try HostInfo( + deviceId: FfiConverterString.read(from: &buf), + name: FfiConverterOptionString.read(from: &buf), + online: FfiConverterBool.read(from: &buf), + capabilities: FfiConverterTypeHostCapabilities.read(from: &buf) ) } - public static func write(_ value: Fade, into buf: inout [UInt8]) { - FfiConverterFloat.write(value.x, into: &buf) - FfiConverterFloat.write(value.y, into: &buf) - FfiConverterFloat.write(value.w, into: &buf) - FfiConverterFloat.write(value.h, into: &buf) - FfiConverterTypeFadeEdge.write(value.edge, into: &buf) - FfiConverterOptionUInt32.write(value.scroller, into: &buf) + public static func write(_ value: HostInfo, into buf: inout [UInt8]) { + FfiConverterString.write(value.deviceId, into: &buf) + FfiConverterOptionString.write(value.name, into: &buf) + FfiConverterBool.write(value.online, into: &buf) + FfiConverterTypeHostCapabilities.write(value.capabilities, into: &buf) } } @@ -6017,30 +7081,29 @@ public struct FfiConverterTypeFade: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFade_lift(_ buf: RustBuffer) throws -> Fade { - return try FfiConverterTypeFade.lift(buf) +public func FfiConverterTypeHostInfo_lift(_ buf: RustBuffer) throws -> HostInfo { + return try FfiConverterTypeHostInfo.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFade_lower(_ value: Fade) -> RustBuffer { - return FfiConverterTypeFade.lower(value) +public func FfiConverterTypeHostInfo_lower(_ value: HostInfo) -> RustBuffer { + return FfiConverterTypeHostInfo.lower(value) } -/** - * A workspace file or folder for `@` mentions. - */ -public struct FileMatch: Equatable, Hashable { - public var path: String - public var isDir: Bool +public struct InputRequest: Equatable, Hashable { + public var entryId: String + public var requestId: String + public var questions: [UserInputQuestion] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(path: String, isDir: Bool) { - self.path = path - self.isDir = isDir + public init(entryId: String, requestId: String, questions: [UserInputQuestion]) { + self.entryId = entryId + self.requestId = requestId + self.questions = questions } @@ -6049,24 +7112,26 @@ public struct FileMatch: Equatable, Hashable { } #if compiler(>=6) -extension FileMatch: Sendable {} +extension InputRequest: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeFileMatch: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FileMatch { +public struct FfiConverterTypeInputRequest: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> InputRequest { return - try FileMatch( - path: FfiConverterString.read(from: &buf), - isDir: FfiConverterBool.read(from: &buf) + try InputRequest( + entryId: FfiConverterString.read(from: &buf), + requestId: FfiConverterString.read(from: &buf), + questions: FfiConverterSequenceTypeUserInputQuestion.read(from: &buf) ) } - public static func write(_ value: FileMatch, into buf: inout [UInt8]) { - FfiConverterString.write(value.path, into: &buf) - FfiConverterBool.write(value.isDir, into: &buf) + public static func write(_ value: InputRequest, into buf: inout [UInt8]) { + FfiConverterString.write(value.entryId, into: &buf) + FfiConverterString.write(value.requestId, into: &buf) + FfiConverterSequenceTypeUserInputQuestion.write(value.questions, into: &buf) } } @@ -6074,29 +7139,38 @@ public struct FfiConverterTypeFileMatch: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFileMatch_lift(_ buf: RustBuffer) throws -> FileMatch { - return try FfiConverterTypeFileMatch.lift(buf) +public func FfiConverterTypeInputRequest_lift(_ buf: RustBuffer) throws -> InputRequest { + return try FfiConverterTypeInputRequest.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFileMatch_lower(_ value: FileMatch) -> RustBuffer { - return FfiConverterTypeFileMatch.lower(value) -} - - -public struct FolderEntry: Equatable, Hashable { - public var name: String - public var isDir: Bool - public var isRepo: Bool +public func FfiConverterTypeInputRequest_lower(_ value: InputRequest) -> RustBuffer { + return FfiConverterTypeInputRequest.lower(value) +} + + +/** + * A tappable link region (one per fragment; multi-line links repeat). + */ +public struct LinkHit: Equatable, Hashable { + public var x: Float + public var y: Float + public var w: Float + public var h: Float + public var url: String + public var scroller: UInt32? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(name: String, isDir: Bool, isRepo: Bool) { - self.name = name - self.isDir = isDir - self.isRepo = isRepo + public init(x: Float, y: Float, w: Float, h: Float, url: String, scroller: UInt32?) { + self.x = x + self.y = y + self.w = w + self.h = h + self.url = url + self.scroller = scroller } @@ -6105,26 +7179,32 @@ public struct FolderEntry: Equatable, Hashable { } #if compiler(>=6) -extension FolderEntry: Sendable {} +extension LinkHit: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeFolderEntry: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FolderEntry { +public struct FfiConverterTypeLinkHit: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LinkHit { return - try FolderEntry( - name: FfiConverterString.read(from: &buf), - isDir: FfiConverterBool.read(from: &buf), - isRepo: FfiConverterBool.read(from: &buf) + try LinkHit( + x: FfiConverterFloat.read(from: &buf), + y: FfiConverterFloat.read(from: &buf), + w: FfiConverterFloat.read(from: &buf), + h: FfiConverterFloat.read(from: &buf), + url: FfiConverterString.read(from: &buf), + scroller: FfiConverterOptionUInt32.read(from: &buf) ) } - public static func write(_ value: FolderEntry, into buf: inout [UInt8]) { - FfiConverterString.write(value.name, into: &buf) - FfiConverterBool.write(value.isDir, into: &buf) - FfiConverterBool.write(value.isRepo, into: &buf) + public static func write(_ value: LinkHit, into buf: inout [UInt8]) { + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.w, into: &buf) + FfiConverterFloat.write(value.h, into: &buf) + FfiConverterString.write(value.url, into: &buf) + FfiConverterOptionUInt32.write(value.scroller, into: &buf) } } @@ -6132,29 +7212,57 @@ public struct FfiConverterTypeFolderEntry: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFolderEntry_lift(_ buf: RustBuffer) throws -> FolderEntry { - return try FfiConverterTypeFolderEntry.lift(buf) +public func FfiConverterTypeLinkHit_lift(_ buf: RustBuffer) throws -> LinkHit { + return try FfiConverterTypeLinkHit.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFolderEntry_lower(_ value: FolderEntry) -> RustBuffer { - return FfiConverterTypeFolderEntry.lower(value) +public func FfiConverterTypeLinkHit_lower(_ value: LinkHit) -> RustBuffer { + return FfiConverterTypeLinkHit.lower(value) } -public struct FolderListing: Equatable, Hashable { - public var path: String - public var entries: [FolderEntry] - public var truncated: Bool +public struct LiveStatus: Equatable, Hashable { + /** + * Display status (an in-flight send reads Working). + */ + public var indicator: ChatIndicator + /** + * A turn is actually running on the host (sends queue/steer). + */ + public var turnRunning: Bool + public var workingSinceMs: Int64? + /** + * The newest transcript entry is streaming. + */ + public var streaming: Bool + /** + * Stop is meaningful. + */ + public var canInterrupt: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(path: String, entries: [FolderEntry], truncated: Bool) { - self.path = path - self.entries = entries - self.truncated = truncated + public init( + /** + * Display status (an in-flight send reads Working). + */indicator: ChatIndicator, + /** + * A turn is actually running on the host (sends queue/steer). + */turnRunning: Bool, workingSinceMs: Int64?, + /** + * The newest transcript entry is streaming. + */streaming: Bool, + /** + * Stop is meaningful. + */canInterrupt: Bool) { + self.indicator = indicator + self.turnRunning = turnRunning + self.workingSinceMs = workingSinceMs + self.streaming = streaming + self.canInterrupt = canInterrupt } @@ -6163,26 +7271,30 @@ public struct FolderListing: Equatable, Hashable { } #if compiler(>=6) -extension FolderListing: Sendable {} +extension LiveStatus: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeFolderListing: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FolderListing { +public struct FfiConverterTypeLiveStatus: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LiveStatus { return - try FolderListing( - path: FfiConverterString.read(from: &buf), - entries: FfiConverterSequenceTypeFolderEntry.read(from: &buf), - truncated: FfiConverterBool.read(from: &buf) + try LiveStatus( + indicator: FfiConverterTypeChatIndicator.read(from: &buf), + turnRunning: FfiConverterBool.read(from: &buf), + workingSinceMs: FfiConverterOptionInt64.read(from: &buf), + streaming: FfiConverterBool.read(from: &buf), + canInterrupt: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: FolderListing, into buf: inout [UInt8]) { - FfiConverterString.write(value.path, into: &buf) - FfiConverterSequenceTypeFolderEntry.write(value.entries, into: &buf) - FfiConverterBool.write(value.truncated, into: &buf) + public static func write(_ value: LiveStatus, into buf: inout [UInt8]) { + FfiConverterTypeChatIndicator.write(value.indicator, into: &buf) + FfiConverterBool.write(value.turnRunning, into: &buf) + FfiConverterOptionInt64.write(value.workingSinceMs, into: &buf) + FfiConverterBool.write(value.streaming, into: &buf) + FfiConverterBool.write(value.canInterrupt, into: &buf) } } @@ -6190,50 +7302,47 @@ public struct FfiConverterTypeFolderListing: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFolderListing_lift(_ buf: RustBuffer) throws -> FolderListing { - return try FfiConverterTypeFolderListing.lift(buf) +public func FfiConverterTypeLiveStatus_lift(_ buf: RustBuffer) throws -> LiveStatus { + return try FfiConverterTypeLiveStatus.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFolderListing_lower(_ value: FolderListing) -> RustBuffer { - return FfiConverterTypeFolderListing.lower(value) +public func FfiConverterTypeLiveStatus_lower(_ value: LiveStatus) -> RustBuffer { + return FfiConverterTypeLiveStatus.lower(value) } -/** - * The Threads page — the desktop sidebar order. - */ -public struct FrontPage: Equatable, Hashable { - /** - * Shared manual pin order. - */ - public var pinned: [SessionRow] +public struct ModelInfo: Equatable, Hashable { + public var id: String + public var label: String + public var description: String? /** - * User sections ("folders"), each in recency order. + * Wire effort values; empty = no effort ladder. */ - public var sections: [SectionView] + public var reasoningLevels: [String] + public var options: [ModelOption] /** - * Everything else, by recency. + * The picker's default effort for this model. */ - public var recent: [SessionRow] + public var defaultReasoning: String? // Default memberwise initializers are never public by default, so we // declare one manually. - public init( - /** - * Shared manual pin order. - */pinned: [SessionRow], + public init(id: String, label: String, description: String?, /** - * User sections ("folders"), each in recency order. - */sections: [SectionView], + * Wire effort values; empty = no effort ladder. + */reasoningLevels: [String], options: [ModelOption], /** - * Everything else, by recency. - */recent: [SessionRow]) { - self.pinned = pinned - self.sections = sections - self.recent = recent + * The picker's default effort for this model. + */defaultReasoning: String?) { + self.id = id + self.label = label + self.description = description + self.reasoningLevels = reasoningLevels + self.options = options + self.defaultReasoning = defaultReasoning } @@ -6242,26 +7351,32 @@ public struct FrontPage: Equatable, Hashable { } #if compiler(>=6) -extension FrontPage: Sendable {} +extension ModelInfo: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeFrontPage: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FrontPage { +public struct FfiConverterTypeModelInfo: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelInfo { return - try FrontPage( - pinned: FfiConverterSequenceTypeSessionRow.read(from: &buf), - sections: FfiConverterSequenceTypeSectionView.read(from: &buf), - recent: FfiConverterSequenceTypeSessionRow.read(from: &buf) + try ModelInfo( + id: FfiConverterString.read(from: &buf), + label: FfiConverterString.read(from: &buf), + description: FfiConverterOptionString.read(from: &buf), + reasoningLevels: FfiConverterSequenceString.read(from: &buf), + options: FfiConverterSequenceTypeModelOption.read(from: &buf), + defaultReasoning: FfiConverterOptionString.read(from: &buf) ) } - public static func write(_ value: FrontPage, into buf: inout [UInt8]) { - FfiConverterSequenceTypeSessionRow.write(value.pinned, into: &buf) - FfiConverterSequenceTypeSectionView.write(value.sections, into: &buf) - FfiConverterSequenceTypeSessionRow.write(value.recent, into: &buf) + public static func write(_ value: ModelInfo, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterString.write(value.label, into: &buf) + FfiConverterOptionString.write(value.description, into: &buf) + FfiConverterSequenceString.write(value.reasoningLevels, into: &buf) + FfiConverterSequenceTypeModelOption.write(value.options, into: &buf) + FfiConverterOptionString.write(value.defaultReasoning, into: &buf) } } @@ -6269,59 +7384,31 @@ public struct FfiConverterTypeFrontPage: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFrontPage_lift(_ buf: RustBuffer) throws -> FrontPage { - return try FfiConverterTypeFrontPage.lift(buf) +public func FfiConverterTypeModelInfo_lift(_ buf: RustBuffer) throws -> ModelInfo { + return try FfiConverterTypeModelInfo.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeFrontPage_lower(_ value: FrontPage) -> RustBuffer { - return FfiConverterTypeFrontPage.lower(value) +public func FfiConverterTypeModelInfo_lower(_ value: ModelInfo) -> RustBuffer { + return FfiConverterTypeModelInfo.lower(value) } -public struct HarnessInfo: Equatable, Hashable { +public struct ModelOption: Equatable, Hashable { public var id: String public var label: String - public var supportsSteering: Bool? - /** - * `step-boundary` / `turn-boundary`. - */ - public var steeringMode: String? - public var reasoningLevels: [String] - public var installed: Bool - public var enabled: Bool? - /** - * Installed and not disabled on the device. - */ - public var offered: Bool - /** - * Steers mid-turn (unknown until a live catalog). - */ - public var midTurnSteering: Bool? + public var choices: [ModelOptionChoice] + public var defaultChoice: String // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, label: String, supportsSteering: Bool?, - /** - * `step-boundary` / `turn-boundary`. - */steeringMode: String?, reasoningLevels: [String], installed: Bool, enabled: Bool?, - /** - * Installed and not disabled on the device. - */offered: Bool, - /** - * Steers mid-turn (unknown until a live catalog). - */midTurnSteering: Bool?) { - self.id = id - self.label = label - self.supportsSteering = supportsSteering - self.steeringMode = steeringMode - self.reasoningLevels = reasoningLevels - self.installed = installed - self.enabled = enabled - self.offered = offered - self.midTurnSteering = midTurnSteering + public init(id: String, label: String, choices: [ModelOptionChoice], defaultChoice: String) { + self.id = id + self.label = label + self.choices = choices + self.defaultChoice = defaultChoice } @@ -6330,38 +7417,28 @@ public struct HarnessInfo: Equatable, Hashable { } #if compiler(>=6) -extension HarnessInfo: Sendable {} +extension ModelOption: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeHarnessInfo: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HarnessInfo { +public struct FfiConverterTypeModelOption: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelOption { return - try HarnessInfo( + try ModelOption( id: FfiConverterString.read(from: &buf), label: FfiConverterString.read(from: &buf), - supportsSteering: FfiConverterOptionBool.read(from: &buf), - steeringMode: FfiConverterOptionString.read(from: &buf), - reasoningLevels: FfiConverterSequenceString.read(from: &buf), - installed: FfiConverterBool.read(from: &buf), - enabled: FfiConverterOptionBool.read(from: &buf), - offered: FfiConverterBool.read(from: &buf), - midTurnSteering: FfiConverterOptionBool.read(from: &buf) + choices: FfiConverterSequenceTypeModelOptionChoice.read(from: &buf), + defaultChoice: FfiConverterString.read(from: &buf) ) } - public static func write(_ value: HarnessInfo, into buf: inout [UInt8]) { + public static func write(_ value: ModelOption, into buf: inout [UInt8]) { FfiConverterString.write(value.id, into: &buf) FfiConverterString.write(value.label, into: &buf) - FfiConverterOptionBool.write(value.supportsSteering, into: &buf) - FfiConverterOptionString.write(value.steeringMode, into: &buf) - FfiConverterSequenceString.write(value.reasoningLevels, into: &buf) - FfiConverterBool.write(value.installed, into: &buf) - FfiConverterOptionBool.write(value.enabled, into: &buf) - FfiConverterBool.write(value.offered, into: &buf) - FfiConverterOptionBool.write(value.midTurnSteering, into: &buf) + FfiConverterSequenceTypeModelOptionChoice.write(value.choices, into: &buf) + FfiConverterString.write(value.defaultChoice, into: &buf) } } @@ -6369,49 +7446,27 @@ public struct FfiConverterTypeHarnessInfo: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeHarnessInfo_lift(_ buf: RustBuffer) throws -> HarnessInfo { - return try FfiConverterTypeHarnessInfo.lift(buf) +public func FfiConverterTypeModelOption_lift(_ buf: RustBuffer) throws -> ModelOption { + return try FfiConverterTypeModelOption.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeHarnessInfo_lower(_ value: HarnessInfo) -> RustBuffer { - return FfiConverterTypeHarnessInfo.lower(value) +public func FfiConverterTypeModelOption_lower(_ value: ModelOption) -> RustBuffer { + return FfiConverterTypeModelOption.lower(value) } -public struct HostCapabilities: Equatable, Hashable { - public var messageQueue: Bool - public var queueActions: Bool - public var queueAttachments: Bool - public var cleanAttachmentText: Bool - public var queueEditLease: Bool - /** - * Host ≥ 0.2.12: attachments ride `pending://` refs. - */ - public var queuedAttachments: Bool - /** - * The chat's harness steers mid-turn (unknown until a live catalog). - */ - public var midTurnSteering: Bool? +public struct ModelOptionChoice: Equatable, Hashable { + public var id: String + public var label: String // Default memberwise initializers are never public by default, so we // declare one manually. - public init(messageQueue: Bool, queueActions: Bool, queueAttachments: Bool, cleanAttachmentText: Bool, queueEditLease: Bool, - /** - * Host ≥ 0.2.12: attachments ride `pending://` refs. - */queuedAttachments: Bool, - /** - * The chat's harness steers mid-turn (unknown until a live catalog). - */midTurnSteering: Bool?) { - self.messageQueue = messageQueue - self.queueActions = queueActions - self.queueAttachments = queueAttachments - self.cleanAttachmentText = cleanAttachmentText - self.queueEditLease = queueEditLease - self.queuedAttachments = queuedAttachments - self.midTurnSteering = midTurnSteering + public init(id: String, label: String) { + self.id = id + self.label = label } @@ -6420,34 +7475,24 @@ public struct HostCapabilities: Equatable, Hashable { } #if compiler(>=6) -extension HostCapabilities: Sendable {} +extension ModelOptionChoice: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeHostCapabilities: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HostCapabilities { +public struct FfiConverterTypeModelOptionChoice: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelOptionChoice { return - try HostCapabilities( - messageQueue: FfiConverterBool.read(from: &buf), - queueActions: FfiConverterBool.read(from: &buf), - queueAttachments: FfiConverterBool.read(from: &buf), - cleanAttachmentText: FfiConverterBool.read(from: &buf), - queueEditLease: FfiConverterBool.read(from: &buf), - queuedAttachments: FfiConverterBool.read(from: &buf), - midTurnSteering: FfiConverterOptionBool.read(from: &buf) + try ModelOptionChoice( + id: FfiConverterString.read(from: &buf), + label: FfiConverterString.read(from: &buf) ) } - public static func write(_ value: HostCapabilities, into buf: inout [UInt8]) { - FfiConverterBool.write(value.messageQueue, into: &buf) - FfiConverterBool.write(value.queueActions, into: &buf) - FfiConverterBool.write(value.queueAttachments, into: &buf) - FfiConverterBool.write(value.cleanAttachmentText, into: &buf) - FfiConverterBool.write(value.queueEditLease, into: &buf) - FfiConverterBool.write(value.queuedAttachments, into: &buf) - FfiConverterOptionBool.write(value.midTurnSteering, into: &buf) + public static func write(_ value: ModelOptionChoice, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterString.write(value.label, into: &buf) } } @@ -6455,31 +7500,45 @@ public struct FfiConverterTypeHostCapabilities: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeHostCapabilities_lift(_ buf: RustBuffer) throws -> HostCapabilities { - return try FfiConverterTypeHostCapabilities.lift(buf) +public func FfiConverterTypeModelOptionChoice_lift(_ buf: RustBuffer) throws -> ModelOptionChoice { + return try FfiConverterTypeModelOptionChoice.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeHostCapabilities_lower(_ value: HostCapabilities) -> RustBuffer { - return FfiConverterTypeHostCapabilities.lower(value) +public func FfiConverterTypeModelOptionChoice_lower(_ value: ModelOptionChoice) -> RustBuffer { + return FfiConverterTypeModelOptionChoice.lower(value) } -public struct HostInfo: Equatable, Hashable { - public var deviceId: String - public var name: String? - public var online: Bool - public var capabilities: HostCapabilities +public struct NewSession: Equatable, Hashable { + public var target: SessionTarget + public var config: ChatConfig? + /** + * Branch stamped from the first frame (picked ref). + */ + public var branch: String? + /** + * Checkout override (an existing worktree's path). + */ + public var cwd: String? + public var title: String? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(deviceId: String, name: String?, online: Bool, capabilities: HostCapabilities) { - self.deviceId = deviceId - self.name = name - self.online = online - self.capabilities = capabilities + public init(target: SessionTarget, config: ChatConfig?, + /** + * Branch stamped from the first frame (picked ref). + */branch: String?, + /** + * Checkout override (an existing worktree's path). + */cwd: String?, title: String?) { + self.target = target + self.config = config + self.branch = branch + self.cwd = cwd + self.title = title } @@ -6488,28 +7547,30 @@ public struct HostInfo: Equatable, Hashable { } #if compiler(>=6) -extension HostInfo: Sendable {} +extension NewSession: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeHostInfo: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HostInfo { +public struct FfiConverterTypeNewSession: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> NewSession { return - try HostInfo( - deviceId: FfiConverterString.read(from: &buf), - name: FfiConverterOptionString.read(from: &buf), - online: FfiConverterBool.read(from: &buf), - capabilities: FfiConverterTypeHostCapabilities.read(from: &buf) + try NewSession( + target: FfiConverterTypeSessionTarget.read(from: &buf), + config: FfiConverterOptionTypeChatConfig.read(from: &buf), + branch: FfiConverterOptionString.read(from: &buf), + cwd: FfiConverterOptionString.read(from: &buf), + title: FfiConverterOptionString.read(from: &buf) ) } - public static func write(_ value: HostInfo, into buf: inout [UInt8]) { - FfiConverterString.write(value.deviceId, into: &buf) - FfiConverterOptionString.write(value.name, into: &buf) - FfiConverterBool.write(value.online, into: &buf) - FfiConverterTypeHostCapabilities.write(value.capabilities, into: &buf) + public static func write(_ value: NewSession, into buf: inout [UInt8]) { + FfiConverterTypeSessionTarget.write(value.target, into: &buf) + FfiConverterOptionTypeChatConfig.write(value.config, into: &buf) + FfiConverterOptionString.write(value.branch, into: &buf) + FfiConverterOptionString.write(value.cwd, into: &buf) + FfiConverterOptionString.write(value.title, into: &buf) } } @@ -6517,29 +7578,35 @@ public struct FfiConverterTypeHostInfo: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeHostInfo_lift(_ buf: RustBuffer) throws -> HostInfo { - return try FfiConverterTypeHostInfo.lift(buf) +public func FfiConverterTypeNewSession_lift(_ buf: RustBuffer) throws -> NewSession { + return try FfiConverterTypeNewSession.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeHostInfo_lower(_ value: HostInfo) -> RustBuffer { - return FfiConverterTypeHostInfo.lower(value) +public func FfiConverterTypeNewSession_lower(_ value: NewSession) -> RustBuffer { + return FfiConverterTypeNewSession.lower(value) } -public struct InputRequest: Equatable, Hashable { - public var entryId: String - public var requestId: String - public var questions: [UserInputQuestion] +public struct OrbFrame: Equatable, Hashable { + /** + * Edge length of the artwork box, in logical points. + */ + public var size: Float + public var lines: [Float] + public var dots: [Float] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(entryId: String, requestId: String, questions: [UserInputQuestion]) { - self.entryId = entryId - self.requestId = requestId - self.questions = questions + public init( + /** + * Edge length of the artwork box, in logical points. + */size: Float, lines: [Float], dots: [Float]) { + self.size = size + self.lines = lines + self.dots = dots } @@ -6548,26 +7615,26 @@ public struct InputRequest: Equatable, Hashable { } #if compiler(>=6) -extension InputRequest: Sendable {} +extension OrbFrame: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeInputRequest: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> InputRequest { +public struct FfiConverterTypeOrbFrame: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> OrbFrame { return - try InputRequest( - entryId: FfiConverterString.read(from: &buf), - requestId: FfiConverterString.read(from: &buf), - questions: FfiConverterSequenceTypeUserInputQuestion.read(from: &buf) + try OrbFrame( + size: FfiConverterFloat.read(from: &buf), + lines: FfiConverterSequenceFloat.read(from: &buf), + dots: FfiConverterSequenceFloat.read(from: &buf) ) } - public static func write(_ value: InputRequest, into buf: inout [UInt8]) { - FfiConverterString.write(value.entryId, into: &buf) - FfiConverterString.write(value.requestId, into: &buf) - FfiConverterSequenceTypeUserInputQuestion.write(value.questions, into: &buf) + public static func write(_ value: OrbFrame, into buf: inout [UInt8]) { + FfiConverterFloat.write(value.size, into: &buf) + FfiConverterSequenceFloat.write(value.lines, into: &buf) + FfiConverterSequenceFloat.write(value.dots, into: &buf) } } @@ -6575,38 +7642,29 @@ public struct FfiConverterTypeInputRequest: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeInputRequest_lift(_ buf: RustBuffer) throws -> InputRequest { - return try FfiConverterTypeInputRequest.lift(buf) +public func FfiConverterTypeOrbFrame_lift(_ buf: RustBuffer) throws -> OrbFrame { + return try FfiConverterTypeOrbFrame.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeInputRequest_lower(_ value: InputRequest) -> RustBuffer { - return FfiConverterTypeInputRequest.lower(value) +public func FfiConverterTypeOrbFrame_lower(_ value: OrbFrame) -> RustBuffer { + return FfiConverterTypeOrbFrame.lower(value) } -/** - * A tappable link region (one per fragment; multi-line links repeat). - */ -public struct LinkHit: Equatable, Hashable { - public var x: Float - public var y: Float - public var w: Float - public var h: Float - public var url: String - public var scroller: UInt32? - - // Default memberwise initializers are never public by default, so we - // declare one manually. - public init(x: Float, y: Float, w: Float, h: Float, url: String, scroller: UInt32?) { - self.x = x - self.y = y - self.w = w - self.h = h - self.url = url - self.scroller = scroller +public struct OutgoingAttachment: Equatable, Hashable { + public var name: String + public var mimeType: String + public var data: Data + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(name: String, mimeType: String, data: Data) { + self.name = name + self.mimeType = mimeType + self.data = data } @@ -6615,32 +7673,26 @@ public struct LinkHit: Equatable, Hashable { } #if compiler(>=6) -extension LinkHit: Sendable {} +extension OutgoingAttachment: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeLinkHit: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LinkHit { +public struct FfiConverterTypeOutgoingAttachment: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> OutgoingAttachment { return - try LinkHit( - x: FfiConverterFloat.read(from: &buf), - y: FfiConverterFloat.read(from: &buf), - w: FfiConverterFloat.read(from: &buf), - h: FfiConverterFloat.read(from: &buf), - url: FfiConverterString.read(from: &buf), - scroller: FfiConverterOptionUInt32.read(from: &buf) + try OutgoingAttachment( + name: FfiConverterString.read(from: &buf), + mimeType: FfiConverterString.read(from: &buf), + data: FfiConverterData.read(from: &buf) ) } - public static func write(_ value: LinkHit, into buf: inout [UInt8]) { - FfiConverterFloat.write(value.x, into: &buf) - FfiConverterFloat.write(value.y, into: &buf) - FfiConverterFloat.write(value.w, into: &buf) - FfiConverterFloat.write(value.h, into: &buf) - FfiConverterString.write(value.url, into: &buf) - FfiConverterOptionUInt32.write(value.scroller, into: &buf) + public static func write(_ value: OutgoingAttachment, into buf: inout [UInt8]) { + FfiConverterString.write(value.name, into: &buf) + FfiConverterString.write(value.mimeType, into: &buf) + FfiConverterData.write(value.data, into: &buf) } } @@ -6648,57 +7700,33 @@ public struct FfiConverterTypeLinkHit: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeLinkHit_lift(_ buf: RustBuffer) throws -> LinkHit { - return try FfiConverterTypeLinkHit.lift(buf) +public func FfiConverterTypeOutgoingAttachment_lift(_ buf: RustBuffer) throws -> OutgoingAttachment { + return try FfiConverterTypeOutgoingAttachment.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeLinkHit_lower(_ value: LinkHit) -> RustBuffer { - return FfiConverterTypeLinkHit.lower(value) +public func FfiConverterTypeOutgoingAttachment_lower(_ value: OutgoingAttachment) -> RustBuffer { + return FfiConverterTypeOutgoingAttachment.lower(value) } -public struct LiveStatus: Equatable, Hashable { - /** - * Display status (an in-flight send reads Working). - */ - public var indicator: ChatIndicator - /** - * A turn is actually running on the host (sends queue/steer). - */ - public var turnRunning: Bool - public var workingSinceMs: Int64? - /** - * The newest transcript entry is streaming. - */ - public var streaming: Bool +public struct ParsedUserMessage: Equatable, Hashable { /** - * Stop is meaningful. + * Visible prompt (trailer + Appshot context stripped). */ - public var canInterrupt: Bool + public var text: String + public var images: [UserImage] // Default memberwise initializers are never public by default, so we // declare one manually. public init( /** - * Display status (an in-flight send reads Working). - */indicator: ChatIndicator, - /** - * A turn is actually running on the host (sends queue/steer). - */turnRunning: Bool, workingSinceMs: Int64?, - /** - * The newest transcript entry is streaming. - */streaming: Bool, - /** - * Stop is meaningful. - */canInterrupt: Bool) { - self.indicator = indicator - self.turnRunning = turnRunning - self.workingSinceMs = workingSinceMs - self.streaming = streaming - self.canInterrupt = canInterrupt + * Visible prompt (trailer + Appshot context stripped). + */text: String, images: [UserImage]) { + self.text = text + self.images = images } @@ -6707,30 +7735,24 @@ public struct LiveStatus: Equatable, Hashable { } #if compiler(>=6) -extension LiveStatus: Sendable {} +extension ParsedUserMessage: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeLiveStatus: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LiveStatus { +public struct FfiConverterTypeParsedUserMessage: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ParsedUserMessage { return - try LiveStatus( - indicator: FfiConverterTypeChatIndicator.read(from: &buf), - turnRunning: FfiConverterBool.read(from: &buf), - workingSinceMs: FfiConverterOptionInt64.read(from: &buf), - streaming: FfiConverterBool.read(from: &buf), - canInterrupt: FfiConverterBool.read(from: &buf) + try ParsedUserMessage( + text: FfiConverterString.read(from: &buf), + images: FfiConverterSequenceTypeUserImage.read(from: &buf) ) } - public static func write(_ value: LiveStatus, into buf: inout [UInt8]) { - FfiConverterTypeChatIndicator.write(value.indicator, into: &buf) - FfiConverterBool.write(value.turnRunning, into: &buf) - FfiConverterOptionInt64.write(value.workingSinceMs, into: &buf) - FfiConverterBool.write(value.streaming, into: &buf) - FfiConverterBool.write(value.canInterrupt, into: &buf) + public static func write(_ value: ParsedUserMessage, into buf: inout [UInt8]) { + FfiConverterString.write(value.text, into: &buf) + FfiConverterSequenceTypeUserImage.write(value.images, into: &buf) } } @@ -6738,47 +7760,49 @@ public struct FfiConverterTypeLiveStatus: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeLiveStatus_lift(_ buf: RustBuffer) throws -> LiveStatus { - return try FfiConverterTypeLiveStatus.lift(buf) +public func FfiConverterTypeParsedUserMessage_lift(_ buf: RustBuffer) throws -> ParsedUserMessage { + return try FfiConverterTypeParsedUserMessage.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeLiveStatus_lower(_ value: LiveStatus) -> RustBuffer { - return FfiConverterTypeLiveStatus.lower(value) +public func FfiConverterTypeParsedUserMessage_lower(_ value: ParsedUserMessage) -> RustBuffer { + return FfiConverterTypeParsedUserMessage.lower(value) } -public struct ModelInfo: Equatable, Hashable { - public var id: String - public var label: String - public var description: String? +public struct PendingSend: Equatable, Hashable { /** - * Wire effort values; empty = no effort ladder. + * Client-minted id (the host's user entry reuses it). */ - public var reasoningLevels: [String] - public var options: [ModelOption] + public var messageId: String /** - * The picker's default effort for this model. + * Raw content (attachment trailer included). */ - public var defaultReasoning: String? + public var text: String + public var visibleText: String + public var images: [String] + public var kind: PendingKind + public var sentAtMs: Int64 + public var state: SendState // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, label: String, description: String?, + public init( /** - * Wire effort values; empty = no effort ladder. - */reasoningLevels: [String], options: [ModelOption], + * Client-minted id (the host's user entry reuses it). + */messageId: String, /** - * The picker's default effort for this model. - */defaultReasoning: String?) { - self.id = id - self.label = label - self.description = description - self.reasoningLevels = reasoningLevels - self.options = options - self.defaultReasoning = defaultReasoning + * Raw content (attachment trailer included). + */text: String, visibleText: String, images: [String], kind: PendingKind, sentAtMs: Int64, state: SendState) { + self.messageId = messageId + self.text = text + self.visibleText = visibleText + self.images = images + self.kind = kind + self.sentAtMs = sentAtMs + self.state = state } @@ -6787,32 +7811,34 @@ public struct ModelInfo: Equatable, Hashable { } #if compiler(>=6) -extension ModelInfo: Sendable {} +extension PendingSend: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeModelInfo: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelInfo { +public struct FfiConverterTypePendingSend: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PendingSend { return - try ModelInfo( - id: FfiConverterString.read(from: &buf), - label: FfiConverterString.read(from: &buf), - description: FfiConverterOptionString.read(from: &buf), - reasoningLevels: FfiConverterSequenceString.read(from: &buf), - options: FfiConverterSequenceTypeModelOption.read(from: &buf), - defaultReasoning: FfiConverterOptionString.read(from: &buf) + try PendingSend( + messageId: FfiConverterString.read(from: &buf), + text: FfiConverterString.read(from: &buf), + visibleText: FfiConverterString.read(from: &buf), + images: FfiConverterSequenceString.read(from: &buf), + kind: FfiConverterTypePendingKind.read(from: &buf), + sentAtMs: FfiConverterInt64.read(from: &buf), + state: FfiConverterTypeSendState.read(from: &buf) ) } - public static func write(_ value: ModelInfo, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.label, into: &buf) - FfiConverterOptionString.write(value.description, into: &buf) - FfiConverterSequenceString.write(value.reasoningLevels, into: &buf) - FfiConverterSequenceTypeModelOption.write(value.options, into: &buf) - FfiConverterOptionString.write(value.defaultReasoning, into: &buf) + public static func write(_ value: PendingSend, into buf: inout [UInt8]) { + FfiConverterString.write(value.messageId, into: &buf) + FfiConverterString.write(value.text, into: &buf) + FfiConverterString.write(value.visibleText, into: &buf) + FfiConverterSequenceString.write(value.images, into: &buf) + FfiConverterTypePendingKind.write(value.kind, into: &buf) + FfiConverterInt64.write(value.sentAtMs, into: &buf) + FfiConverterTypeSendState.write(value.state, into: &buf) } } @@ -6820,31 +7846,35 @@ public struct FfiConverterTypeModelInfo: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeModelInfo_lift(_ buf: RustBuffer) throws -> ModelInfo { - return try FfiConverterTypeModelInfo.lift(buf) +public func FfiConverterTypePendingSend_lift(_ buf: RustBuffer) throws -> PendingSend { + return try FfiConverterTypePendingSend.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeModelInfo_lower(_ value: ModelInfo) -> RustBuffer { - return FfiConverterTypeModelInfo.lower(value) +public func FfiConverterTypePendingSend_lower(_ value: PendingSend) -> RustBuffer { + return FfiConverterTypePendingSend.lower(value) } -public struct ModelOption: Equatable, Hashable { +public struct ProjectRef: Equatable, Hashable { public var id: String - public var label: String - public var choices: [ModelOptionChoice] - public var defaultChoice: String + public var name: String + /** + * Stable palette slot (0..PROJECT_COLOR_COUNT). + */ + public var colorIndex: UInt32 // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, label: String, choices: [ModelOptionChoice], defaultChoice: String) { + public init(id: String, name: String, + /** + * Stable palette slot (0..PROJECT_COLOR_COUNT). + */colorIndex: UInt32) { self.id = id - self.label = label - self.choices = choices - self.defaultChoice = defaultChoice + self.name = name + self.colorIndex = colorIndex } @@ -6853,28 +7883,26 @@ public struct ModelOption: Equatable, Hashable { } #if compiler(>=6) -extension ModelOption: Sendable {} +extension ProjectRef: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeModelOption: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelOption { +public struct FfiConverterTypeProjectRef: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ProjectRef { return - try ModelOption( + try ProjectRef( id: FfiConverterString.read(from: &buf), - label: FfiConverterString.read(from: &buf), - choices: FfiConverterSequenceTypeModelOptionChoice.read(from: &buf), - defaultChoice: FfiConverterString.read(from: &buf) + name: FfiConverterString.read(from: &buf), + colorIndex: FfiConverterUInt32.read(from: &buf) ) } - public static func write(_ value: ModelOption, into buf: inout [UInt8]) { + public static func write(_ value: ProjectRef, into buf: inout [UInt8]) { FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.label, into: &buf) - FfiConverterSequenceTypeModelOptionChoice.write(value.choices, into: &buf) - FfiConverterString.write(value.defaultChoice, into: &buf) + FfiConverterString.write(value.name, into: &buf) + FfiConverterUInt32.write(value.colorIndex, into: &buf) } } @@ -6882,27 +7910,75 @@ public struct FfiConverterTypeModelOption: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeModelOption_lift(_ buf: RustBuffer) throws -> ModelOption { - return try FfiConverterTypeModelOption.lift(buf) +public func FfiConverterTypeProjectRef_lift(_ buf: RustBuffer) throws -> ProjectRef { + return try FfiConverterTypeProjectRef.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeModelOption_lower(_ value: ModelOption) -> RustBuffer { - return FfiConverterTypeModelOption.lower(value) +public func FfiConverterTypeProjectRef_lower(_ value: ProjectRef) -> RustBuffer { + return FfiConverterTypeProjectRef.lower(value) } -public struct ModelOptionChoice: Equatable, Hashable { +public struct ProjectView: Equatable, Hashable { public var id: String - public var label: String + public var name: String + public var path: String + public var colorIndex: UInt32 + public var deviceId: String + public var deviceName: String? + public var deviceOnline: Bool + public var gitDetected: Bool + public var createdAtMs: Int64 + /** + * Shared by every checkout of one repository, on any device. + */ + public var groupKey: String + /** + * The repository group's name, shared like `color_index`. + */ + public var groupName: String + /** + * Most urgent indicator among its active sessions. + */ + public var indicator: ChatIndicator + public var unseenCount: UInt32 + /** + * Active sessions, recency order. + */ + public var sessions: [SessionRow] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, label: String) { + public init(id: String, name: String, path: String, colorIndex: UInt32, deviceId: String, deviceName: String?, deviceOnline: Bool, gitDetected: Bool, createdAtMs: Int64, + /** + * Shared by every checkout of one repository, on any device. + */groupKey: String, + /** + * The repository group's name, shared like `color_index`. + */groupName: String, + /** + * Most urgent indicator among its active sessions. + */indicator: ChatIndicator, unseenCount: UInt32, + /** + * Active sessions, recency order. + */sessions: [SessionRow]) { self.id = id - self.label = label + self.name = name + self.path = path + self.colorIndex = colorIndex + self.deviceId = deviceId + self.deviceName = deviceName + self.deviceOnline = deviceOnline + self.gitDetected = gitDetected + self.createdAtMs = createdAtMs + self.groupKey = groupKey + self.groupName = groupName + self.indicator = indicator + self.unseenCount = unseenCount + self.sessions = sessions } @@ -6911,24 +7987,48 @@ public struct ModelOptionChoice: Equatable, Hashable { } #if compiler(>=6) -extension ModelOptionChoice: Sendable {} +extension ProjectView: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeModelOptionChoice: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelOptionChoice { +public struct FfiConverterTypeProjectView: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ProjectView { return - try ModelOptionChoice( + try ProjectView( id: FfiConverterString.read(from: &buf), - label: FfiConverterString.read(from: &buf) + name: FfiConverterString.read(from: &buf), + path: FfiConverterString.read(from: &buf), + colorIndex: FfiConverterUInt32.read(from: &buf), + deviceId: FfiConverterString.read(from: &buf), + deviceName: FfiConverterOptionString.read(from: &buf), + deviceOnline: FfiConverterBool.read(from: &buf), + gitDetected: FfiConverterBool.read(from: &buf), + createdAtMs: FfiConverterInt64.read(from: &buf), + groupKey: FfiConverterString.read(from: &buf), + groupName: FfiConverterString.read(from: &buf), + indicator: FfiConverterTypeChatIndicator.read(from: &buf), + unseenCount: FfiConverterUInt32.read(from: &buf), + sessions: FfiConverterSequenceTypeSessionRow.read(from: &buf) ) } - public static func write(_ value: ModelOptionChoice, into buf: inout [UInt8]) { + public static func write(_ value: ProjectView, into buf: inout [UInt8]) { FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.label, into: &buf) + FfiConverterString.write(value.name, into: &buf) + FfiConverterString.write(value.path, into: &buf) + FfiConverterUInt32.write(value.colorIndex, into: &buf) + FfiConverterString.write(value.deviceId, into: &buf) + FfiConverterOptionString.write(value.deviceName, into: &buf) + FfiConverterBool.write(value.deviceOnline, into: &buf) + FfiConverterBool.write(value.gitDetected, into: &buf) + FfiConverterInt64.write(value.createdAtMs, into: &buf) + FfiConverterString.write(value.groupKey, into: &buf) + FfiConverterString.write(value.groupName, into: &buf) + FfiConverterTypeChatIndicator.write(value.indicator, into: &buf) + FfiConverterUInt32.write(value.unseenCount, into: &buf) + FfiConverterSequenceTypeSessionRow.write(value.sessions, into: &buf) } } @@ -6936,45 +8036,37 @@ public struct FfiConverterTypeModelOptionChoice: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeModelOptionChoice_lift(_ buf: RustBuffer) throws -> ModelOptionChoice { - return try FfiConverterTypeModelOptionChoice.lift(buf) +public func FfiConverterTypeProjectView_lift(_ buf: RustBuffer) throws -> ProjectView { + return try FfiConverterTypeProjectView.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeModelOptionChoice_lower(_ value: ModelOptionChoice) -> RustBuffer { - return FfiConverterTypeModelOptionChoice.lower(value) +public func FfiConverterTypeProjectView_lower(_ value: ProjectView) -> RustBuffer { + return FfiConverterTypeProjectView.lower(value) } -public struct NewSession: Equatable, Hashable { - public var target: SessionTarget - public var config: ChatConfig? - /** - * Branch stamped from the first frame (picked ref). - */ - public var branch: String? - /** - * Checkout override (an existing worktree's path). - */ - public var cwd: String? - public var title: String? +public struct PullRequest: Equatable, Hashable { + public var provider: String + public var number: UInt64 + public var title: String + public var url: String + public var state: PullRequestState + public var baseRef: String + public var headRef: String // Default memberwise initializers are never public by default, so we // declare one manually. - public init(target: SessionTarget, config: ChatConfig?, - /** - * Branch stamped from the first frame (picked ref). - */branch: String?, - /** - * Checkout override (an existing worktree's path). - */cwd: String?, title: String?) { - self.target = target - self.config = config - self.branch = branch - self.cwd = cwd + public init(provider: String, number: UInt64, title: String, url: String, state: PullRequestState, baseRef: String, headRef: String) { + self.provider = provider + self.number = number self.title = title + self.url = url + self.state = state + self.baseRef = baseRef + self.headRef = headRef } @@ -6983,30 +8075,34 @@ public struct NewSession: Equatable, Hashable { } #if compiler(>=6) -extension NewSession: Sendable {} +extension PullRequest: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeNewSession: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> NewSession { +public struct FfiConverterTypePullRequest: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PullRequest { return - try NewSession( - target: FfiConverterTypeSessionTarget.read(from: &buf), - config: FfiConverterOptionTypeChatConfig.read(from: &buf), - branch: FfiConverterOptionString.read(from: &buf), - cwd: FfiConverterOptionString.read(from: &buf), - title: FfiConverterOptionString.read(from: &buf) + try PullRequest( + provider: FfiConverterString.read(from: &buf), + number: FfiConverterUInt64.read(from: &buf), + title: FfiConverterString.read(from: &buf), + url: FfiConverterString.read(from: &buf), + state: FfiConverterTypePullRequestState.read(from: &buf), + baseRef: FfiConverterString.read(from: &buf), + headRef: FfiConverterString.read(from: &buf) ) } - public static func write(_ value: NewSession, into buf: inout [UInt8]) { - FfiConverterTypeSessionTarget.write(value.target, into: &buf) - FfiConverterOptionTypeChatConfig.write(value.config, into: &buf) - FfiConverterOptionString.write(value.branch, into: &buf) - FfiConverterOptionString.write(value.cwd, into: &buf) - FfiConverterOptionString.write(value.title, into: &buf) + public static func write(_ value: PullRequest, into buf: inout [UInt8]) { + FfiConverterString.write(value.provider, into: &buf) + FfiConverterUInt64.write(value.number, into: &buf) + FfiConverterString.write(value.title, into: &buf) + FfiConverterString.write(value.url, into: &buf) + FfiConverterTypePullRequestState.write(value.state, into: &buf) + FfiConverterString.write(value.baseRef, into: &buf) + FfiConverterString.write(value.headRef, into: &buf) } } @@ -7014,35 +8110,29 @@ public struct FfiConverterTypeNewSession: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeNewSession_lift(_ buf: RustBuffer) throws -> NewSession { - return try FfiConverterTypeNewSession.lift(buf) +public func FfiConverterTypePullRequest_lift(_ buf: RustBuffer) throws -> PullRequest { + return try FfiConverterTypePullRequest.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeNewSession_lower(_ value: NewSession) -> RustBuffer { - return FfiConverterTypeNewSession.lower(value) +public func FfiConverterTypePullRequest_lower(_ value: PullRequest) -> RustBuffer { + return FfiConverterTypePullRequest.lower(value) } -public struct OrbFrame: Equatable, Hashable { - /** - * Edge length of the artwork box, in logical points. - */ - public var size: Float - public var lines: [Float] - public var dots: [Float] +public struct PullRequestGroups: Equatable, Hashable { + public var `open`: [SessionRow] + public var merged: [SessionRow] + public var closed: [SessionRow] // Default memberwise initializers are never public by default, so we // declare one manually. - public init( - /** - * Edge length of the artwork box, in logical points. - */size: Float, lines: [Float], dots: [Float]) { - self.size = size - self.lines = lines - self.dots = dots + public init(`open`: [SessionRow], merged: [SessionRow], closed: [SessionRow]) { + self.`open` = `open` + self.merged = merged + self.closed = closed } @@ -7051,26 +8141,26 @@ public struct OrbFrame: Equatable, Hashable { } #if compiler(>=6) -extension OrbFrame: Sendable {} +extension PullRequestGroups: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeOrbFrame: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> OrbFrame { +public struct FfiConverterTypePullRequestGroups: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PullRequestGroups { return - try OrbFrame( - size: FfiConverterFloat.read(from: &buf), - lines: FfiConverterSequenceFloat.read(from: &buf), - dots: FfiConverterSequenceFloat.read(from: &buf) + try PullRequestGroups( + open: FfiConverterSequenceTypeSessionRow.read(from: &buf), + merged: FfiConverterSequenceTypeSessionRow.read(from: &buf), + closed: FfiConverterSequenceTypeSessionRow.read(from: &buf) ) } - public static func write(_ value: OrbFrame, into buf: inout [UInt8]) { - FfiConverterFloat.write(value.size, into: &buf) - FfiConverterSequenceFloat.write(value.lines, into: &buf) - FfiConverterSequenceFloat.write(value.dots, into: &buf) + public static func write(_ value: PullRequestGroups, into buf: inout [UInt8]) { + FfiConverterSequenceTypeSessionRow.write(value.`open`, into: &buf) + FfiConverterSequenceTypeSessionRow.write(value.merged, into: &buf) + FfiConverterSequenceTypeSessionRow.write(value.closed, into: &buf) } } @@ -7078,29 +8168,32 @@ public struct FfiConverterTypeOrbFrame: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeOrbFrame_lift(_ buf: RustBuffer) throws -> OrbFrame { - return try FfiConverterTypeOrbFrame.lift(buf) +public func FfiConverterTypePullRequestGroups_lift(_ buf: RustBuffer) throws -> PullRequestGroups { + return try FfiConverterTypePullRequestGroups.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeOrbFrame_lower(_ value: OrbFrame) -> RustBuffer { - return FfiConverterTypeOrbFrame.lower(value) +public func FfiConverterTypePullRequestGroups_lower(_ value: PullRequestGroups) -> RustBuffer { + return FfiConverterTypePullRequestGroups.lower(value) } -public struct OutgoingAttachment: Equatable, Hashable { - public var name: String - public var mimeType: String - public var data: Data +/** + * Which session notifications this device wants. + */ +public struct PushPrefs: Equatable, Hashable { + public var done: Bool + public var input: Bool + public var failed: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(name: String, mimeType: String, data: Data) { - self.name = name - self.mimeType = mimeType - self.data = data + public init(done: Bool, input: Bool, failed: Bool) { + self.done = done + self.input = input + self.failed = failed } @@ -7109,26 +8202,26 @@ public struct OutgoingAttachment: Equatable, Hashable { } #if compiler(>=6) -extension OutgoingAttachment: Sendable {} +extension PushPrefs: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeOutgoingAttachment: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> OutgoingAttachment { - return - try OutgoingAttachment( - name: FfiConverterString.read(from: &buf), - mimeType: FfiConverterString.read(from: &buf), - data: FfiConverterData.read(from: &buf) +public struct FfiConverterTypePushPrefs: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PushPrefs { + return + try PushPrefs( + done: FfiConverterBool.read(from: &buf), + input: FfiConverterBool.read(from: &buf), + failed: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: OutgoingAttachment, into buf: inout [UInt8]) { - FfiConverterString.write(value.name, into: &buf) - FfiConverterString.write(value.mimeType, into: &buf) - FfiConverterData.write(value.data, into: &buf) + public static func write(_ value: PushPrefs, into buf: inout [UInt8]) { + FfiConverterBool.write(value.done, into: &buf) + FfiConverterBool.write(value.input, into: &buf) + FfiConverterBool.write(value.failed, into: &buf) } } @@ -7136,33 +8229,39 @@ public struct FfiConverterTypeOutgoingAttachment: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeOutgoingAttachment_lift(_ buf: RustBuffer) throws -> OutgoingAttachment { - return try FfiConverterTypeOutgoingAttachment.lift(buf) +public func FfiConverterTypePushPrefs_lift(_ buf: RustBuffer) throws -> PushPrefs { + return try FfiConverterTypePushPrefs.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeOutgoingAttachment_lower(_ value: OutgoingAttachment) -> RustBuffer { - return FfiConverterTypeOutgoingAttachment.lower(value) +public func FfiConverterTypePushPrefs_lower(_ value: PushPrefs) -> RustBuffer { + return FfiConverterTypePushPrefs.lower(value) } -public struct ParsedUserMessage: Equatable, Hashable { +public struct QueueEditLease: Equatable, Hashable { + public var rowId: String + public var leaseId: String /** - * Visible prompt (trailer + Appshot context stripped). + * Row text at lease time. */ public var text: String - public var images: [UserImage] + public var baseTextHash: String + public var expiresAtMs: Int64 // Default memberwise initializers are never public by default, so we // declare one manually. - public init( + public init(rowId: String, leaseId: String, /** - * Visible prompt (trailer + Appshot context stripped). - */text: String, images: [UserImage]) { + * Row text at lease time. + */text: String, baseTextHash: String, expiresAtMs: Int64) { + self.rowId = rowId + self.leaseId = leaseId self.text = text - self.images = images + self.baseTextHash = baseTextHash + self.expiresAtMs = expiresAtMs } @@ -7171,24 +8270,30 @@ public struct ParsedUserMessage: Equatable, Hashable { } #if compiler(>=6) -extension ParsedUserMessage: Sendable {} +extension QueueEditLease: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeParsedUserMessage: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ParsedUserMessage { +public struct FfiConverterTypeQueueEditLease: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> QueueEditLease { return - try ParsedUserMessage( + try QueueEditLease( + rowId: FfiConverterString.read(from: &buf), + leaseId: FfiConverterString.read(from: &buf), text: FfiConverterString.read(from: &buf), - images: FfiConverterSequenceTypeUserImage.read(from: &buf) + baseTextHash: FfiConverterString.read(from: &buf), + expiresAtMs: FfiConverterInt64.read(from: &buf) ) } - public static func write(_ value: ParsedUserMessage, into buf: inout [UInt8]) { + public static func write(_ value: QueueEditLease, into buf: inout [UInt8]) { + FfiConverterString.write(value.rowId, into: &buf) + FfiConverterString.write(value.leaseId, into: &buf) FfiConverterString.write(value.text, into: &buf) - FfiConverterSequenceTypeUserImage.write(value.images, into: &buf) + FfiConverterString.write(value.baseTextHash, into: &buf) + FfiConverterInt64.write(value.expiresAtMs, into: &buf) } } @@ -7196,49 +8301,63 @@ public struct FfiConverterTypeParsedUserMessage: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeParsedUserMessage_lift(_ buf: RustBuffer) throws -> ParsedUserMessage { - return try FfiConverterTypeParsedUserMessage.lift(buf) +public func FfiConverterTypeQueueEditLease_lift(_ buf: RustBuffer) throws -> QueueEditLease { + return try FfiConverterTypeQueueEditLease.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeParsedUserMessage_lower(_ value: ParsedUserMessage) -> RustBuffer { - return FfiConverterTypeParsedUserMessage.lower(value) +public func FfiConverterTypeQueueEditLease_lower(_ value: QueueEditLease) -> RustBuffer { + return FfiConverterTypeQueueEditLease.lower(value) } -public struct PendingSend: Equatable, Hashable { +public struct QueueItem: Equatable, Hashable { + public var id: String /** - * Client-minted id (the host's user entry reuses it). + * Raw row text (what the host will send). */ - public var messageId: String + public var text: String /** - * Raw content (attachment trailer included). + * User-visible text (trailer/Appshot context stripped). */ - public var text: String public var visibleText: String - public var images: [String] - public var kind: PendingKind - public var sentAtMs: Int64 - public var state: SendState + public var attachments: [String] + public var holdForTurnEnd: Bool + public var issuedBy: String + public var fromThisDevice: Bool + public var issuedAtMs: Int64 + public var editedAtMs: Int64? + public var gate: QueueGate? + /** + * A Send-now / Remove is awaiting the host's acknowledgement. + */ + public var actionPending: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init( + public init(id: String, /** - * Client-minted id (the host's user entry reuses it). - */messageId: String, + * Raw row text (what the host will send). + */text: String, /** - * Raw content (attachment trailer included). - */text: String, visibleText: String, images: [String], kind: PendingKind, sentAtMs: Int64, state: SendState) { - self.messageId = messageId + * User-visible text (trailer/Appshot context stripped). + */visibleText: String, attachments: [String], holdForTurnEnd: Bool, issuedBy: String, fromThisDevice: Bool, issuedAtMs: Int64, editedAtMs: Int64?, gate: QueueGate?, + /** + * A Send-now / Remove is awaiting the host's acknowledgement. + */actionPending: Bool) { + self.id = id self.text = text self.visibleText = visibleText - self.images = images - self.kind = kind - self.sentAtMs = sentAtMs - self.state = state + self.attachments = attachments + self.holdForTurnEnd = holdForTurnEnd + self.issuedBy = issuedBy + self.fromThisDevice = fromThisDevice + self.issuedAtMs = issuedAtMs + self.editedAtMs = editedAtMs + self.gate = gate + self.actionPending = actionPending } @@ -7247,34 +8366,42 @@ public struct PendingSend: Equatable, Hashable { } #if compiler(>=6) -extension PendingSend: Sendable {} +extension QueueItem: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypePendingSend: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PendingSend { +public struct FfiConverterTypeQueueItem: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> QueueItem { return - try PendingSend( - messageId: FfiConverterString.read(from: &buf), + try QueueItem( + id: FfiConverterString.read(from: &buf), text: FfiConverterString.read(from: &buf), visibleText: FfiConverterString.read(from: &buf), - images: FfiConverterSequenceString.read(from: &buf), - kind: FfiConverterTypePendingKind.read(from: &buf), - sentAtMs: FfiConverterInt64.read(from: &buf), - state: FfiConverterTypeSendState.read(from: &buf) + attachments: FfiConverterSequenceString.read(from: &buf), + holdForTurnEnd: FfiConverterBool.read(from: &buf), + issuedBy: FfiConverterString.read(from: &buf), + fromThisDevice: FfiConverterBool.read(from: &buf), + issuedAtMs: FfiConverterInt64.read(from: &buf), + editedAtMs: FfiConverterOptionInt64.read(from: &buf), + gate: FfiConverterOptionTypeQueueGate.read(from: &buf), + actionPending: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: PendingSend, into buf: inout [UInt8]) { - FfiConverterString.write(value.messageId, into: &buf) + public static func write(_ value: QueueItem, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) FfiConverterString.write(value.text, into: &buf) FfiConverterString.write(value.visibleText, into: &buf) - FfiConverterSequenceString.write(value.images, into: &buf) - FfiConverterTypePendingKind.write(value.kind, into: &buf) - FfiConverterInt64.write(value.sentAtMs, into: &buf) - FfiConverterTypeSendState.write(value.state, into: &buf) + FfiConverterSequenceString.write(value.attachments, into: &buf) + FfiConverterBool.write(value.holdForTurnEnd, into: &buf) + FfiConverterString.write(value.issuedBy, into: &buf) + FfiConverterBool.write(value.fromThisDevice, into: &buf) + FfiConverterInt64.write(value.issuedAtMs, into: &buf) + FfiConverterOptionInt64.write(value.editedAtMs, into: &buf) + FfiConverterOptionTypeQueueGate.write(value.gate, into: &buf) + FfiConverterBool.write(value.actionPending, into: &buf) } } @@ -7282,35 +8409,41 @@ public struct FfiConverterTypePendingSend: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePendingSend_lift(_ buf: RustBuffer) throws -> PendingSend { - return try FfiConverterTypePendingSend.lift(buf) +public func FfiConverterTypeQueueItem_lift(_ buf: RustBuffer) throws -> QueueItem { + return try FfiConverterTypeQueueItem.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePendingSend_lower(_ value: PendingSend) -> RustBuffer { - return FfiConverterTypePendingSend.lower(value) +public func FfiConverterTypeQueueItem_lower(_ value: QueueItem) -> RustBuffer { + return FfiConverterTypeQueueItem.lower(value) } -public struct ProjectRef: Equatable, Hashable { - public var id: String +public struct RepoRef: Equatable, Hashable { public var name: String /** - * Stable palette slot (0..PROJECT_COLOR_COUNT). + * Checked out in the repo's main folder right now. */ - public var colorIndex: UInt32 + public var current: Bool + /** + * Linked worktree this branch is checked out in, if any. + */ + public var worktreePath: String? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, name: String, + public init(name: String, /** - * Stable palette slot (0..PROJECT_COLOR_COUNT). - */colorIndex: UInt32) { - self.id = id + * Checked out in the repo's main folder right now. + */current: Bool, + /** + * Linked worktree this branch is checked out in, if any. + */worktreePath: String?) { self.name = name - self.colorIndex = colorIndex + self.current = current + self.worktreePath = worktreePath } @@ -7319,102 +8452,61 @@ public struct ProjectRef: Equatable, Hashable { } #if compiler(>=6) -extension ProjectRef: Sendable {} +extension RepoRef: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeProjectRef: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ProjectRef { +public struct FfiConverterTypeRepoRef: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RepoRef { return - try ProjectRef( - id: FfiConverterString.read(from: &buf), + try RepoRef( name: FfiConverterString.read(from: &buf), - colorIndex: FfiConverterUInt32.read(from: &buf) + current: FfiConverterBool.read(from: &buf), + worktreePath: FfiConverterOptionString.read(from: &buf) ) } - public static func write(_ value: ProjectRef, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) + public static func write(_ value: RepoRef, into buf: inout [UInt8]) { FfiConverterString.write(value.name, into: &buf) - FfiConverterUInt32.write(value.colorIndex, into: &buf) + FfiConverterBool.write(value.current, into: &buf) + FfiConverterOptionString.write(value.worktreePath, into: &buf) } -} - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeProjectRef_lift(_ buf: RustBuffer) throws -> ProjectRef { - return try FfiConverterTypeProjectRef.lift(buf) -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeProjectRef_lower(_ value: ProjectRef) -> RustBuffer { - return FfiConverterTypeProjectRef.lower(value) -} - - -public struct ProjectView: Equatable, Hashable { - public var id: String - public var name: String - public var path: String - public var colorIndex: UInt32 - public var deviceId: String - public var deviceName: String? - public var deviceOnline: Bool - public var gitDetected: Bool - public var createdAtMs: Int64 - /** - * Shared by every checkout of one repository, on any device. - */ - public var groupKey: String - /** - * The repository group's name, shared like `color_index`. - */ - public var groupName: String - /** - * Most urgent indicator among its active sessions. - */ - public var indicator: ChatIndicator - public var unseenCount: UInt32 - /** - * Active sessions, recency order. - */ - public var sessions: [SessionRow] +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeRepoRef_lift(_ buf: RustBuffer) throws -> RepoRef { + return try FfiConverterTypeRepoRef.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeRepoRef_lower(_ value: RepoRef) -> RustBuffer { + return FfiConverterTypeRepoRef.lower(value) +} + + +/** + * sRGB, straight alpha, 0–1. + */ +public struct Rgba: Equatable, Hashable { + public var r: Float + public var g: Float + public var b: Float + public var a: Float // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, name: String, path: String, colorIndex: UInt32, deviceId: String, deviceName: String?, deviceOnline: Bool, gitDetected: Bool, createdAtMs: Int64, - /** - * Shared by every checkout of one repository, on any device. - */groupKey: String, - /** - * The repository group's name, shared like `color_index`. - */groupName: String, - /** - * Most urgent indicator among its active sessions. - */indicator: ChatIndicator, unseenCount: UInt32, - /** - * Active sessions, recency order. - */sessions: [SessionRow]) { - self.id = id - self.name = name - self.path = path - self.colorIndex = colorIndex - self.deviceId = deviceId - self.deviceName = deviceName - self.deviceOnline = deviceOnline - self.gitDetected = gitDetected - self.createdAtMs = createdAtMs - self.groupKey = groupKey - self.groupName = groupName - self.indicator = indicator - self.unseenCount = unseenCount - self.sessions = sessions + public init(r: Float, g: Float, b: Float, a: Float) { + self.r = r + self.g = g + self.b = b + self.a = a } @@ -7423,48 +8515,28 @@ public struct ProjectView: Equatable, Hashable { } #if compiler(>=6) -extension ProjectView: Sendable {} +extension Rgba: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeProjectView: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ProjectView { +public struct FfiConverterTypeRgba: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Rgba { return - try ProjectView( - id: FfiConverterString.read(from: &buf), - name: FfiConverterString.read(from: &buf), - path: FfiConverterString.read(from: &buf), - colorIndex: FfiConverterUInt32.read(from: &buf), - deviceId: FfiConverterString.read(from: &buf), - deviceName: FfiConverterOptionString.read(from: &buf), - deviceOnline: FfiConverterBool.read(from: &buf), - gitDetected: FfiConverterBool.read(from: &buf), - createdAtMs: FfiConverterInt64.read(from: &buf), - groupKey: FfiConverterString.read(from: &buf), - groupName: FfiConverterString.read(from: &buf), - indicator: FfiConverterTypeChatIndicator.read(from: &buf), - unseenCount: FfiConverterUInt32.read(from: &buf), - sessions: FfiConverterSequenceTypeSessionRow.read(from: &buf) + try Rgba( + r: FfiConverterFloat.read(from: &buf), + g: FfiConverterFloat.read(from: &buf), + b: FfiConverterFloat.read(from: &buf), + a: FfiConverterFloat.read(from: &buf) ) } - public static func write(_ value: ProjectView, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.name, into: &buf) - FfiConverterString.write(value.path, into: &buf) - FfiConverterUInt32.write(value.colorIndex, into: &buf) - FfiConverterString.write(value.deviceId, into: &buf) - FfiConverterOptionString.write(value.deviceName, into: &buf) - FfiConverterBool.write(value.deviceOnline, into: &buf) - FfiConverterBool.write(value.gitDetected, into: &buf) - FfiConverterInt64.write(value.createdAtMs, into: &buf) - FfiConverterString.write(value.groupKey, into: &buf) - FfiConverterString.write(value.groupName, into: &buf) - FfiConverterTypeChatIndicator.write(value.indicator, into: &buf) - FfiConverterUInt32.write(value.unseenCount, into: &buf) - FfiConverterSequenceTypeSessionRow.write(value.sessions, into: &buf) + public static func write(_ value: Rgba, into buf: inout [UInt8]) { + FfiConverterFloat.write(value.r, into: &buf) + FfiConverterFloat.write(value.g, into: &buf) + FfiConverterFloat.write(value.b, into: &buf) + FfiConverterFloat.write(value.a, into: &buf) } } @@ -7472,37 +8544,35 @@ public struct FfiConverterTypeProjectView: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeProjectView_lift(_ buf: RustBuffer) throws -> ProjectView { - return try FfiConverterTypeProjectView.lift(buf) +public func FfiConverterTypeRgba_lift(_ buf: RustBuffer) throws -> Rgba { + return try FfiConverterTypeRgba.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeProjectView_lower(_ value: ProjectView) -> RustBuffer { - return FfiConverterTypeProjectView.lower(value) +public func FfiConverterTypeRgba_lower(_ value: Rgba) -> RustBuffer { + return FfiConverterTypeRgba.lower(value) } -public struct PullRequest: Equatable, Hashable { - public var provider: String - public var number: UInt64 - public var title: String - public var url: String - public var state: PullRequestState - public var baseRef: String - public var headRef: String +public struct RoomState: Equatable, Hashable { + public var connected: Bool + public var retryAtMs: Int64? + /** + * Graced degradation for this chat. + */ + public var degraded: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(provider: String, number: UInt64, title: String, url: String, state: PullRequestState, baseRef: String, headRef: String) { - self.provider = provider - self.number = number - self.title = title - self.url = url - self.state = state - self.baseRef = baseRef - self.headRef = headRef + public init(connected: Bool, retryAtMs: Int64?, + /** + * Graced degradation for this chat. + */degraded: Bool) { + self.connected = connected + self.retryAtMs = retryAtMs + self.degraded = degraded } @@ -7511,34 +8581,26 @@ public struct PullRequest: Equatable, Hashable { } #if compiler(>=6) -extension PullRequest: Sendable {} +extension RoomState: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypePullRequest: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PullRequest { +public struct FfiConverterTypeRoomState: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RoomState { return - try PullRequest( - provider: FfiConverterString.read(from: &buf), - number: FfiConverterUInt64.read(from: &buf), - title: FfiConverterString.read(from: &buf), - url: FfiConverterString.read(from: &buf), - state: FfiConverterTypePullRequestState.read(from: &buf), - baseRef: FfiConverterString.read(from: &buf), - headRef: FfiConverterString.read(from: &buf) + try RoomState( + connected: FfiConverterBool.read(from: &buf), + retryAtMs: FfiConverterOptionInt64.read(from: &buf), + degraded: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: PullRequest, into buf: inout [UInt8]) { - FfiConverterString.write(value.provider, into: &buf) - FfiConverterUInt64.write(value.number, into: &buf) - FfiConverterString.write(value.title, into: &buf) - FfiConverterString.write(value.url, into: &buf) - FfiConverterTypePullRequestState.write(value.state, into: &buf) - FfiConverterString.write(value.baseRef, into: &buf) - FfiConverterString.write(value.headRef, into: &buf) + public static func write(_ value: RoomState, into buf: inout [UInt8]) { + FfiConverterBool.write(value.connected, into: &buf) + FfiConverterOptionInt64.write(value.retryAtMs, into: &buf) + FfiConverterBool.write(value.degraded, into: &buf) } } @@ -7546,29 +8608,56 @@ public struct FfiConverterTypePullRequest: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePullRequest_lift(_ buf: RustBuffer) throws -> PullRequest { - return try FfiConverterTypePullRequest.lift(buf) +public func FfiConverterTypeRoomState_lift(_ buf: RustBuffer) throws -> RoomState { + return try FfiConverterTypeRoomState.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePullRequest_lower(_ value: PullRequest) -> RustBuffer { - return FfiConverterTypePullRequest.lower(value) +public func FfiConverterTypeRoomState_lower(_ value: RoomState) -> RustBuffer { + return FfiConverterTypeRoomState.lower(value) } -public struct PullRequestGroups: Equatable, Hashable { - public var `open`: [SessionRow] - public var merged: [SessionRow] - public var closed: [SessionRow] +/** + * Everything needed to paint a row at its current width. + */ +public struct RowDisplay: Equatable, Hashable { + public var key: UInt64 + public var version: UInt64 + public var width: Float + public var height: Float + public var text: String + public var runs: [TextRun] + public var boxes: [BoxPrim] + public var links: [LinkHit] + public var scrollers: [Scroller] + public var widgets: [Widget] + public var fades: [Fade] + /** + * Plain text for copy / accessibility. + */ + public var copyText: String // Default memberwise initializers are never public by default, so we // declare one manually. - public init(`open`: [SessionRow], merged: [SessionRow], closed: [SessionRow]) { - self.`open` = `open` - self.merged = merged - self.closed = closed + public init(key: UInt64, version: UInt64, width: Float, height: Float, text: String, runs: [TextRun], boxes: [BoxPrim], links: [LinkHit], scrollers: [Scroller], widgets: [Widget], fades: [Fade], + /** + * Plain text for copy / accessibility. + */copyText: String) { + self.key = key + self.version = version + self.width = width + self.height = height + self.text = text + self.runs = runs + self.boxes = boxes + self.links = links + self.scrollers = scrollers + self.widgets = widgets + self.fades = fades + self.copyText = copyText } @@ -7577,26 +8666,44 @@ public struct PullRequestGroups: Equatable, Hashable { } #if compiler(>=6) -extension PullRequestGroups: Sendable {} +extension RowDisplay: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypePullRequestGroups: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PullRequestGroups { +public struct FfiConverterTypeRowDisplay: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RowDisplay { return - try PullRequestGroups( - open: FfiConverterSequenceTypeSessionRow.read(from: &buf), - merged: FfiConverterSequenceTypeSessionRow.read(from: &buf), - closed: FfiConverterSequenceTypeSessionRow.read(from: &buf) + try RowDisplay( + key: FfiConverterUInt64.read(from: &buf), + version: FfiConverterUInt64.read(from: &buf), + width: FfiConverterFloat.read(from: &buf), + height: FfiConverterFloat.read(from: &buf), + text: FfiConverterString.read(from: &buf), + runs: FfiConverterSequenceTypeTextRun.read(from: &buf), + boxes: FfiConverterSequenceTypeBoxPrim.read(from: &buf), + links: FfiConverterSequenceTypeLinkHit.read(from: &buf), + scrollers: FfiConverterSequenceTypeScroller.read(from: &buf), + widgets: FfiConverterSequenceTypeWidget.read(from: &buf), + fades: FfiConverterSequenceTypeFade.read(from: &buf), + copyText: FfiConverterString.read(from: &buf) ) } - public static func write(_ value: PullRequestGroups, into buf: inout [UInt8]) { - FfiConverterSequenceTypeSessionRow.write(value.`open`, into: &buf) - FfiConverterSequenceTypeSessionRow.write(value.merged, into: &buf) - FfiConverterSequenceTypeSessionRow.write(value.closed, into: &buf) + public static func write(_ value: RowDisplay, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.key, into: &buf) + FfiConverterUInt64.write(value.version, into: &buf) + FfiConverterFloat.write(value.width, into: &buf) + FfiConverterFloat.write(value.height, into: &buf) + FfiConverterString.write(value.text, into: &buf) + FfiConverterSequenceTypeTextRun.write(value.runs, into: &buf) + FfiConverterSequenceTypeBoxPrim.write(value.boxes, into: &buf) + FfiConverterSequenceTypeLinkHit.write(value.links, into: &buf) + FfiConverterSequenceTypeScroller.write(value.scrollers, into: &buf) + FfiConverterSequenceTypeWidget.write(value.widgets, into: &buf) + FfiConverterSequenceTypeFade.write(value.fades, into: &buf) + FfiConverterString.write(value.copyText, into: &buf) } } @@ -7604,32 +8711,44 @@ public struct FfiConverterTypePullRequestGroups: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePullRequestGroups_lift(_ buf: RustBuffer) throws -> PullRequestGroups { - return try FfiConverterTypePullRequestGroups.lift(buf) +public func FfiConverterTypeRowDisplay_lift(_ buf: RustBuffer) throws -> RowDisplay { + return try FfiConverterTypeRowDisplay.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePullRequestGroups_lower(_ value: PullRequestGroups) -> RustBuffer { - return FfiConverterTypePullRequestGroups.lower(value) +public func FfiConverterTypeRowDisplay_lower(_ value: RowDisplay) -> RustBuffer { + return FfiConverterTypeRowDisplay.lower(value) } /** - * Which session notifications this device wants. + * A row's position in a frame. */ -public struct PushPrefs: Equatable, Hashable { - public var done: Bool - public var input: Bool - public var failed: Bool +public struct RowPlacement: Equatable, Hashable { + public var index: UInt32 + public var key: UInt64 + /** + * Changes whenever the row's painted content may change (not its width). + */ + public var version: UInt64 + public var y: Float + public var height: Float + public var kind: RowKind // Default memberwise initializers are never public by default, so we // declare one manually. - public init(done: Bool, input: Bool, failed: Bool) { - self.done = done - self.input = input - self.failed = failed + public init(index: UInt32, key: UInt64, + /** + * Changes whenever the row's painted content may change (not its width). + */version: UInt64, y: Float, height: Float, kind: RowKind) { + self.index = index + self.key = key + self.version = version + self.y = y + self.height = height + self.kind = kind } @@ -7638,26 +8757,32 @@ public struct PushPrefs: Equatable, Hashable { } #if compiler(>=6) -extension PushPrefs: Sendable {} +extension RowPlacement: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypePushPrefs: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PushPrefs { +public struct FfiConverterTypeRowPlacement: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RowPlacement { return - try PushPrefs( - done: FfiConverterBool.read(from: &buf), - input: FfiConverterBool.read(from: &buf), - failed: FfiConverterBool.read(from: &buf) + try RowPlacement( + index: FfiConverterUInt32.read(from: &buf), + key: FfiConverterUInt64.read(from: &buf), + version: FfiConverterUInt64.read(from: &buf), + y: FfiConverterFloat.read(from: &buf), + height: FfiConverterFloat.read(from: &buf), + kind: FfiConverterTypeRowKind.read(from: &buf) ) } - public static func write(_ value: PushPrefs, into buf: inout [UInt8]) { - FfiConverterBool.write(value.done, into: &buf) - FfiConverterBool.write(value.input, into: &buf) - FfiConverterBool.write(value.failed, into: &buf) + public static func write(_ value: RowPlacement, into buf: inout [UInt8]) { + FfiConverterUInt32.write(value.index, into: &buf) + FfiConverterUInt64.write(value.key, into: &buf) + FfiConverterUInt64.write(value.version, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.height, into: &buf) + FfiConverterTypeRowKind.write(value.kind, into: &buf) } } @@ -7665,39 +8790,38 @@ public struct FfiConverterTypePushPrefs: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePushPrefs_lift(_ buf: RustBuffer) throws -> PushPrefs { - return try FfiConverterTypePushPrefs.lift(buf) +public func FfiConverterTypeRowPlacement_lift(_ buf: RustBuffer) throws -> RowPlacement { + return try FfiConverterTypeRowPlacement.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypePushPrefs_lower(_ value: PushPrefs) -> RustBuffer { - return FfiConverterTypePushPrefs.lower(value) +public func FfiConverterTypeRowPlacement_lower(_ value: RowPlacement) -> RustBuffer { + return FfiConverterTypeRowPlacement.lower(value) } -public struct QueueEditLease: Equatable, Hashable { - public var rowId: String - public var leaseId: String - /** - * Row text at lease time. - */ - public var text: String - public var baseTextHash: String - public var expiresAtMs: Int64 +/** + * A horizontally scrolling viewport (code blocks, wide tables). Primitives + * that reference it are positioned in its content coordinates, whose origin + * is the viewport's top-left. + */ +public struct Scroller: Equatable, Hashable { + public var x: Float + public var y: Float + public var w: Float + public var h: Float + public var contentWidth: Float // Default memberwise initializers are never public by default, so we // declare one manually. - public init(rowId: String, leaseId: String, - /** - * Row text at lease time. - */text: String, baseTextHash: String, expiresAtMs: Int64) { - self.rowId = rowId - self.leaseId = leaseId - self.text = text - self.baseTextHash = baseTextHash - self.expiresAtMs = expiresAtMs + public init(x: Float, y: Float, w: Float, h: Float, contentWidth: Float) { + self.x = x + self.y = y + self.w = w + self.h = h + self.contentWidth = contentWidth } @@ -7706,30 +8830,30 @@ public struct QueueEditLease: Equatable, Hashable { } #if compiler(>=6) -extension QueueEditLease: Sendable {} +extension Scroller: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeQueueEditLease: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> QueueEditLease { +public struct FfiConverterTypeScroller: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Scroller { return - try QueueEditLease( - rowId: FfiConverterString.read(from: &buf), - leaseId: FfiConverterString.read(from: &buf), - text: FfiConverterString.read(from: &buf), - baseTextHash: FfiConverterString.read(from: &buf), - expiresAtMs: FfiConverterInt64.read(from: &buf) + try Scroller( + x: FfiConverterFloat.read(from: &buf), + y: FfiConverterFloat.read(from: &buf), + w: FfiConverterFloat.read(from: &buf), + h: FfiConverterFloat.read(from: &buf), + contentWidth: FfiConverterFloat.read(from: &buf) ) } - public static func write(_ value: QueueEditLease, into buf: inout [UInt8]) { - FfiConverterString.write(value.rowId, into: &buf) - FfiConverterString.write(value.leaseId, into: &buf) - FfiConverterString.write(value.text, into: &buf) - FfiConverterString.write(value.baseTextHash, into: &buf) - FfiConverterInt64.write(value.expiresAtMs, into: &buf) + public static func write(_ value: Scroller, into buf: inout [UInt8]) { + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.w, into: &buf) + FfiConverterFloat.write(value.h, into: &buf) + FfiConverterFloat.write(value.contentWidth, into: &buf) } } @@ -7737,63 +8861,35 @@ public struct FfiConverterTypeQueueEditLease: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeQueueEditLease_lift(_ buf: RustBuffer) throws -> QueueEditLease { - return try FfiConverterTypeQueueEditLease.lift(buf) +public func FfiConverterTypeScroller_lift(_ buf: RustBuffer) throws -> Scroller { + return try FfiConverterTypeScroller.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeQueueEditLease_lower(_ value: QueueEditLease) -> RustBuffer { - return FfiConverterTypeQueueEditLease.lower(value) +public func FfiConverterTypeScroller_lower(_ value: Scroller) -> RustBuffer { + return FfiConverterTypeScroller.lower(value) } -public struct QueueItem: Equatable, Hashable { - public var id: String - /** - * Raw row text (what the host will send). - */ - public var text: String - /** - * User-visible text (trailer/Appshot context stripped). - */ - public var visibleText: String - public var attachments: [String] - public var holdForTurnEnd: Bool - public var issuedBy: String - public var fromThisDevice: Bool - public var issuedAtMs: Int64 - public var editedAtMs: Int64? - public var gate: QueueGate? +public struct SearchHit: Equatable, Hashable { + public var session: SessionRow + public var score: UInt32 /** - * A Send-now / Remove is awaiting the host's acknowledgement. + * The strongest field that matched. */ - public var actionPending: Bool + public var field: SearchField // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, - /** - * Raw row text (what the host will send). - */text: String, - /** - * User-visible text (trailer/Appshot context stripped). - */visibleText: String, attachments: [String], holdForTurnEnd: Bool, issuedBy: String, fromThisDevice: Bool, issuedAtMs: Int64, editedAtMs: Int64?, gate: QueueGate?, + public init(session: SessionRow, score: UInt32, /** - * A Send-now / Remove is awaiting the host's acknowledgement. - */actionPending: Bool) { - self.id = id - self.text = text - self.visibleText = visibleText - self.attachments = attachments - self.holdForTurnEnd = holdForTurnEnd - self.issuedBy = issuedBy - self.fromThisDevice = fromThisDevice - self.issuedAtMs = issuedAtMs - self.editedAtMs = editedAtMs - self.gate = gate - self.actionPending = actionPending + * The strongest field that matched. + */field: SearchField) { + self.session = session + self.score = score + self.field = field } @@ -7802,42 +8898,26 @@ public struct QueueItem: Equatable, Hashable { } #if compiler(>=6) -extension QueueItem: Sendable {} +extension SearchHit: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeQueueItem: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> QueueItem { +public struct FfiConverterTypeSearchHit: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SearchHit { return - try QueueItem( - id: FfiConverterString.read(from: &buf), - text: FfiConverterString.read(from: &buf), - visibleText: FfiConverterString.read(from: &buf), - attachments: FfiConverterSequenceString.read(from: &buf), - holdForTurnEnd: FfiConverterBool.read(from: &buf), - issuedBy: FfiConverterString.read(from: &buf), - fromThisDevice: FfiConverterBool.read(from: &buf), - issuedAtMs: FfiConverterInt64.read(from: &buf), - editedAtMs: FfiConverterOptionInt64.read(from: &buf), - gate: FfiConverterOptionTypeQueueGate.read(from: &buf), - actionPending: FfiConverterBool.read(from: &buf) + try SearchHit( + session: FfiConverterTypeSessionRow.read(from: &buf), + score: FfiConverterUInt32.read(from: &buf), + field: FfiConverterTypeSearchField.read(from: &buf) ) } - public static func write(_ value: QueueItem, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.text, into: &buf) - FfiConverterString.write(value.visibleText, into: &buf) - FfiConverterSequenceString.write(value.attachments, into: &buf) - FfiConverterBool.write(value.holdForTurnEnd, into: &buf) - FfiConverterString.write(value.issuedBy, into: &buf) - FfiConverterBool.write(value.fromThisDevice, into: &buf) - FfiConverterInt64.write(value.issuedAtMs, into: &buf) - FfiConverterOptionInt64.write(value.editedAtMs, into: &buf) - FfiConverterOptionTypeQueueGate.write(value.gate, into: &buf) - FfiConverterBool.write(value.actionPending, into: &buf) + public static func write(_ value: SearchHit, into buf: inout [UInt8]) { + FfiConverterTypeSessionRow.write(value.session, into: &buf) + FfiConverterUInt32.write(value.score, into: &buf) + FfiConverterTypeSearchField.write(value.field, into: &buf) } } @@ -7845,41 +8925,37 @@ public struct FfiConverterTypeQueueItem: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeQueueItem_lift(_ buf: RustBuffer) throws -> QueueItem { - return try FfiConverterTypeQueueItem.lift(buf) +public func FfiConverterTypeSearchHit_lift(_ buf: RustBuffer) throws -> SearchHit { + return try FfiConverterTypeSearchHit.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeQueueItem_lower(_ value: QueueItem) -> RustBuffer { - return FfiConverterTypeQueueItem.lower(value) +public func FfiConverterTypeSearchHit_lower(_ value: SearchHit) -> RustBuffer { + return FfiConverterTypeSearchHit.lower(value) } -public struct RepoRef: Equatable, Hashable { +public struct SectionView: Equatable, Hashable { + public var id: String public var name: String + public var collapsed: Bool /** - * Checked out in the repo's main folder right now. - */ - public var current: Bool - /** - * Linked worktree this branch is checked out in, if any. + * Recency order (pinned sessions excluded). */ - public var worktreePath: String? + public var sessions: [SessionRow] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(name: String, - /** - * Checked out in the repo's main folder right now. - */current: Bool, + public init(id: String, name: String, collapsed: Bool, /** - * Linked worktree this branch is checked out in, if any. - */worktreePath: String?) { + * Recency order (pinned sessions excluded). + */sessions: [SessionRow]) { + self.id = id self.name = name - self.current = current - self.worktreePath = worktreePath + self.collapsed = collapsed + self.sessions = sessions } @@ -7888,26 +8964,28 @@ public struct RepoRef: Equatable, Hashable { } #if compiler(>=6) -extension RepoRef: Sendable {} +extension SectionView: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeRepoRef: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RepoRef { +public struct FfiConverterTypeSectionView: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SectionView { return - try RepoRef( + try SectionView( + id: FfiConverterString.read(from: &buf), name: FfiConverterString.read(from: &buf), - current: FfiConverterBool.read(from: &buf), - worktreePath: FfiConverterOptionString.read(from: &buf) + collapsed: FfiConverterBool.read(from: &buf), + sessions: FfiConverterSequenceTypeSessionRow.read(from: &buf) ) } - public static func write(_ value: RepoRef, into buf: inout [UInt8]) { + public static func write(_ value: SectionView, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) FfiConverterString.write(value.name, into: &buf) - FfiConverterBool.write(value.current, into: &buf) - FfiConverterOptionString.write(value.worktreePath, into: &buf) + FfiConverterBool.write(value.collapsed, into: &buf) + FfiConverterSequenceTypeSessionRow.write(value.sessions, into: &buf) } } @@ -7915,35 +8993,37 @@ public struct FfiConverterTypeRepoRef: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRepoRef_lift(_ buf: RustBuffer) throws -> RepoRef { - return try FfiConverterTypeRepoRef.lift(buf) +public func FfiConverterTypeSectionView_lift(_ buf: RustBuffer) throws -> SectionView { + return try FfiConverterTypeSectionView.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRepoRef_lower(_ value: RepoRef) -> RustBuffer { - return FfiConverterTypeRepoRef.lower(value) +public func FfiConverterTypeSectionView_lower(_ value: SectionView) -> RustBuffer { + return FfiConverterTypeSectionView.lower(value) } -public struct RoomState: Equatable, Hashable { - public var connected: Bool - public var retryAtMs: Int64? +public struct SendRequest: Equatable, Hashable { + public var text: String + public var attachments: [OutgoingAttachment] /** - * Graced degradation for this chat. + * Isolated host worktree for this run (new-session "New worktree"). */ - public var degraded: Bool + public var worktree: WorktreeSpec? + public var busy: BusyPolicy // Default memberwise initializers are never public by default, so we // declare one manually. - public init(connected: Bool, retryAtMs: Int64?, + public init(text: String, attachments: [OutgoingAttachment], /** - * Graced degradation for this chat. - */degraded: Bool) { - self.connected = connected - self.retryAtMs = retryAtMs - self.degraded = degraded + * Isolated host worktree for this run (new-session "New worktree"). + */worktree: WorktreeSpec?, busy: BusyPolicy) { + self.text = text + self.attachments = attachments + self.worktree = worktree + self.busy = busy } @@ -7952,26 +9032,28 @@ public struct RoomState: Equatable, Hashable { } #if compiler(>=6) -extension RoomState: Sendable {} +extension SendRequest: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeRoomState: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RoomState { +public struct FfiConverterTypeSendRequest: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SendRequest { return - try RoomState( - connected: FfiConverterBool.read(from: &buf), - retryAtMs: FfiConverterOptionInt64.read(from: &buf), - degraded: FfiConverterBool.read(from: &buf) + try SendRequest( + text: FfiConverterString.read(from: &buf), + attachments: FfiConverterSequenceTypeOutgoingAttachment.read(from: &buf), + worktree: FfiConverterOptionTypeWorktreeSpec.read(from: &buf), + busy: FfiConverterTypeBusyPolicy.read(from: &buf) ) } - public static func write(_ value: RoomState, into buf: inout [UInt8]) { - FfiConverterBool.write(value.connected, into: &buf) - FfiConverterOptionInt64.write(value.retryAtMs, into: &buf) - FfiConverterBool.write(value.degraded, into: &buf) + public static func write(_ value: SendRequest, into buf: inout [UInt8]) { + FfiConverterString.write(value.text, into: &buf) + FfiConverterSequenceTypeOutgoingAttachment.write(value.attachments, into: &buf) + FfiConverterOptionTypeWorktreeSpec.write(value.worktree, into: &buf) + FfiConverterTypeBusyPolicy.write(value.busy, into: &buf) } } @@ -7979,102 +9061,234 @@ public struct FfiConverterTypeRoomState: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRoomState_lift(_ buf: RustBuffer) throws -> RoomState { - return try FfiConverterTypeRoomState.lift(buf) +public func FfiConverterTypeSendRequest_lift(_ buf: RustBuffer) throws -> SendRequest { + return try FfiConverterTypeSendRequest.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRoomState_lower(_ value: RoomState) -> RustBuffer { - return FfiConverterTypeRoomState.lower(value) +public func FfiConverterTypeSendRequest_lower(_ value: SendRequest) -> RustBuffer { + return FfiConverterTypeSendRequest.lower(value) } /** - * Everything needed to paint a row at its current width. + * One session row, as every list renders it. */ -public struct RowDisplay: Equatable, Hashable { - public var key: UInt64 - public var version: UInt64 - public var width: Float - public var height: Float - public var text: String - public var runs: [TextRun] - public var boxes: [BoxPrim] - public var links: [LinkHit] - public var scrollers: [Scroller] - public var widgets: [Widget] - public var fades: [Fade] +public struct SessionRow: Equatable, Hashable { + public var id: String /** - * Plain text for copy / accessibility. + * Content hash — equal ⇒ nothing to redraw. */ - public var copyText: String + public var revision: UInt64 + /** + * Display title ("New session" when untitled). + */ + public var title: String + public var hasTitle: Bool + public var preview: String? + /** + * `None` = projectless (host's home folder). + */ + public var project: ProjectRef? + public var deviceId: String + public var deviceName: String? + public var deviceOnline: Bool + /** + * Wire harness id (`claude-code`). + */ + public var harness: String? + public var harnessLabel: String? + public var model: String? + public var modelLabel: String? + /** + * Wire effort (`xhigh`). + */ + public var reasoning: String? + public var branch: String? + public var cwd: String? + /** + * Staleness-gated (45s) live status; an in-flight send reads Working. + */ + public var indicator: ChatIndicator + /** + * Host-reported status only (no local-send override). + */ + public var hostIndicator: ChatIndicator + /** + * Run start of the live turn while Working/AwaitingInput. + */ + public var workingSinceMs: Int64? + public var lastActivityMs: Int64 + /** + * `now` / `34m` / `4h` / `2d` at derive time. + */ + public var timeLabel: String + public var createdAtMs: Int64 + public var unseen: Bool + public var archived: Bool + public var pinned: Bool + public var sectionId: String? + public var pullRequest: PullRequest? + /** + * Oldest unadopted send from this device (open sessions only). + */ + public var sendState: SendState? + public var parentChatId: String? + /** + * 2 = chat2; 1 = legacy (not dialable). + */ + public var roomGen: UInt32 // Default memberwise initializers are never public by default, so we // declare one manually. - public init(key: UInt64, version: UInt64, width: Float, height: Float, text: String, runs: [TextRun], boxes: [BoxPrim], links: [LinkHit], scrollers: [Scroller], widgets: [Widget], fades: [Fade], + public init(id: String, /** - * Plain text for copy / accessibility. - */copyText: String) { - self.key = key - self.version = version - self.width = width - self.height = height - self.text = text - self.runs = runs - self.boxes = boxes - self.links = links - self.scrollers = scrollers - self.widgets = widgets - self.fades = fades - self.copyText = copyText + * Content hash — equal ⇒ nothing to redraw. + */revision: UInt64, + /** + * Display title ("New session" when untitled). + */title: String, hasTitle: Bool, preview: String?, + /** + * `None` = projectless (host's home folder). + */project: ProjectRef?, deviceId: String, deviceName: String?, deviceOnline: Bool, + /** + * Wire harness id (`claude-code`). + */harness: String?, harnessLabel: String?, model: String?, modelLabel: String?, + /** + * Wire effort (`xhigh`). + */reasoning: String?, branch: String?, cwd: String?, + /** + * Staleness-gated (45s) live status; an in-flight send reads Working. + */indicator: ChatIndicator, + /** + * Host-reported status only (no local-send override). + */hostIndicator: ChatIndicator, + /** + * Run start of the live turn while Working/AwaitingInput. + */workingSinceMs: Int64?, lastActivityMs: Int64, + /** + * `now` / `34m` / `4h` / `2d` at derive time. + */timeLabel: String, createdAtMs: Int64, unseen: Bool, archived: Bool, pinned: Bool, sectionId: String?, pullRequest: PullRequest?, + /** + * Oldest unadopted send from this device (open sessions only). + */sendState: SendState?, parentChatId: String?, + /** + * 2 = chat2; 1 = legacy (not dialable). + */roomGen: UInt32) { + self.id = id + self.revision = revision + self.title = title + self.hasTitle = hasTitle + self.preview = preview + self.project = project + self.deviceId = deviceId + self.deviceName = deviceName + self.deviceOnline = deviceOnline + self.harness = harness + self.harnessLabel = harnessLabel + self.model = model + self.modelLabel = modelLabel + self.reasoning = reasoning + self.branch = branch + self.cwd = cwd + self.indicator = indicator + self.hostIndicator = hostIndicator + self.workingSinceMs = workingSinceMs + self.lastActivityMs = lastActivityMs + self.timeLabel = timeLabel + self.createdAtMs = createdAtMs + self.unseen = unseen + self.archived = archived + self.pinned = pinned + self.sectionId = sectionId + self.pullRequest = pullRequest + self.sendState = sendState + self.parentChatId = parentChatId + self.roomGen = roomGen } } - -#if compiler(>=6) -extension RowDisplay: Sendable {} -#endif - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public struct FfiConverterTypeRowDisplay: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RowDisplay { - return - try RowDisplay( - key: FfiConverterUInt64.read(from: &buf), - version: FfiConverterUInt64.read(from: &buf), - width: FfiConverterFloat.read(from: &buf), - height: FfiConverterFloat.read(from: &buf), - text: FfiConverterString.read(from: &buf), - runs: FfiConverterSequenceTypeTextRun.read(from: &buf), - boxes: FfiConverterSequenceTypeBoxPrim.read(from: &buf), - links: FfiConverterSequenceTypeLinkHit.read(from: &buf), - scrollers: FfiConverterSequenceTypeScroller.read(from: &buf), - widgets: FfiConverterSequenceTypeWidget.read(from: &buf), - fades: FfiConverterSequenceTypeFade.read(from: &buf), - copyText: FfiConverterString.read(from: &buf) + +#if compiler(>=6) +extension SessionRow: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeSessionRow: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SessionRow { + return + try SessionRow( + id: FfiConverterString.read(from: &buf), + revision: FfiConverterUInt64.read(from: &buf), + title: FfiConverterString.read(from: &buf), + hasTitle: FfiConverterBool.read(from: &buf), + preview: FfiConverterOptionString.read(from: &buf), + project: FfiConverterOptionTypeProjectRef.read(from: &buf), + deviceId: FfiConverterString.read(from: &buf), + deviceName: FfiConverterOptionString.read(from: &buf), + deviceOnline: FfiConverterBool.read(from: &buf), + harness: FfiConverterOptionString.read(from: &buf), + harnessLabel: FfiConverterOptionString.read(from: &buf), + model: FfiConverterOptionString.read(from: &buf), + modelLabel: FfiConverterOptionString.read(from: &buf), + reasoning: FfiConverterOptionString.read(from: &buf), + branch: FfiConverterOptionString.read(from: &buf), + cwd: FfiConverterOptionString.read(from: &buf), + indicator: FfiConverterTypeChatIndicator.read(from: &buf), + hostIndicator: FfiConverterTypeChatIndicator.read(from: &buf), + workingSinceMs: FfiConverterOptionInt64.read(from: &buf), + lastActivityMs: FfiConverterInt64.read(from: &buf), + timeLabel: FfiConverterString.read(from: &buf), + createdAtMs: FfiConverterInt64.read(from: &buf), + unseen: FfiConverterBool.read(from: &buf), + archived: FfiConverterBool.read(from: &buf), + pinned: FfiConverterBool.read(from: &buf), + sectionId: FfiConverterOptionString.read(from: &buf), + pullRequest: FfiConverterOptionTypePullRequest.read(from: &buf), + sendState: FfiConverterOptionTypeSendState.read(from: &buf), + parentChatId: FfiConverterOptionString.read(from: &buf), + roomGen: FfiConverterUInt32.read(from: &buf) ) } - public static func write(_ value: RowDisplay, into buf: inout [UInt8]) { - FfiConverterUInt64.write(value.key, into: &buf) - FfiConverterUInt64.write(value.version, into: &buf) - FfiConverterFloat.write(value.width, into: &buf) - FfiConverterFloat.write(value.height, into: &buf) - FfiConverterString.write(value.text, into: &buf) - FfiConverterSequenceTypeTextRun.write(value.runs, into: &buf) - FfiConverterSequenceTypeBoxPrim.write(value.boxes, into: &buf) - FfiConverterSequenceTypeLinkHit.write(value.links, into: &buf) - FfiConverterSequenceTypeScroller.write(value.scrollers, into: &buf) - FfiConverterSequenceTypeWidget.write(value.widgets, into: &buf) - FfiConverterSequenceTypeFade.write(value.fades, into: &buf) - FfiConverterString.write(value.copyText, into: &buf) + public static func write(_ value: SessionRow, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterUInt64.write(value.revision, into: &buf) + FfiConverterString.write(value.title, into: &buf) + FfiConverterBool.write(value.hasTitle, into: &buf) + FfiConverterOptionString.write(value.preview, into: &buf) + FfiConverterOptionTypeProjectRef.write(value.project, into: &buf) + FfiConverterString.write(value.deviceId, into: &buf) + FfiConverterOptionString.write(value.deviceName, into: &buf) + FfiConverterBool.write(value.deviceOnline, into: &buf) + FfiConverterOptionString.write(value.harness, into: &buf) + FfiConverterOptionString.write(value.harnessLabel, into: &buf) + FfiConverterOptionString.write(value.model, into: &buf) + FfiConverterOptionString.write(value.modelLabel, into: &buf) + FfiConverterOptionString.write(value.reasoning, into: &buf) + FfiConverterOptionString.write(value.branch, into: &buf) + FfiConverterOptionString.write(value.cwd, into: &buf) + FfiConverterTypeChatIndicator.write(value.indicator, into: &buf) + FfiConverterTypeChatIndicator.write(value.hostIndicator, into: &buf) + FfiConverterOptionInt64.write(value.workingSinceMs, into: &buf) + FfiConverterInt64.write(value.lastActivityMs, into: &buf) + FfiConverterString.write(value.timeLabel, into: &buf) + FfiConverterInt64.write(value.createdAtMs, into: &buf) + FfiConverterBool.write(value.unseen, into: &buf) + FfiConverterBool.write(value.archived, into: &buf) + FfiConverterBool.write(value.pinned, into: &buf) + FfiConverterOptionString.write(value.sectionId, into: &buf) + FfiConverterOptionTypePullRequest.write(value.pullRequest, into: &buf) + FfiConverterOptionTypeSendState.write(value.sendState, into: &buf) + FfiConverterOptionString.write(value.parentChatId, into: &buf) + FfiConverterUInt32.write(value.roomGen, into: &buf) } } @@ -8082,44 +9296,34 @@ public struct FfiConverterTypeRowDisplay: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRowDisplay_lift(_ buf: RustBuffer) throws -> RowDisplay { - return try FfiConverterTypeRowDisplay.lift(buf) +public func FfiConverterTypeSessionRow_lift(_ buf: RustBuffer) throws -> SessionRow { + return try FfiConverterTypeSessionRow.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRowDisplay_lower(_ value: RowDisplay) -> RustBuffer { - return FfiConverterTypeRowDisplay.lower(value) +public func FfiConverterTypeSessionRow_lower(_ value: SessionRow) -> RustBuffer { + return FfiConverterTypeSessionRow.lower(value) } /** - * A row's position in a frame. + * A style the platform must be able to draw: `id` → (face, size). */ -public struct RowPlacement: Equatable, Hashable { - public var index: UInt32 - public var key: UInt64 - /** - * Changes whenever the row's painted content may change (not its width). - */ - public var version: UInt64 - public var y: Float - public var height: Float - public var kind: RowKind +public struct StyleDesc: Equatable, Hashable { + public var id: UInt16 + public var face: FaceRole + public var size: Float + public var ligatures: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(index: UInt32, key: UInt64, - /** - * Changes whenever the row's painted content may change (not its width). - */version: UInt64, y: Float, height: Float, kind: RowKind) { - self.index = index - self.key = key - self.version = version - self.y = y - self.height = height - self.kind = kind + public init(id: UInt16, face: FaceRole, size: Float, ligatures: Bool) { + self.id = id + self.face = face + self.size = size + self.ligatures = ligatures } @@ -8128,32 +9332,28 @@ public struct RowPlacement: Equatable, Hashable { } #if compiler(>=6) -extension RowPlacement: Sendable {} +extension StyleDesc: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeRowPlacement: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> RowPlacement { +public struct FfiConverterTypeStyleDesc: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> StyleDesc { return - try RowPlacement( - index: FfiConverterUInt32.read(from: &buf), - key: FfiConverterUInt64.read(from: &buf), - version: FfiConverterUInt64.read(from: &buf), - y: FfiConverterFloat.read(from: &buf), - height: FfiConverterFloat.read(from: &buf), - kind: FfiConverterTypeRowKind.read(from: &buf) + try StyleDesc( + id: FfiConverterUInt16.read(from: &buf), + face: FfiConverterTypeFaceRole.read(from: &buf), + size: FfiConverterFloat.read(from: &buf), + ligatures: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: RowPlacement, into buf: inout [UInt8]) { - FfiConverterUInt32.write(value.index, into: &buf) - FfiConverterUInt64.write(value.key, into: &buf) - FfiConverterUInt64.write(value.version, into: &buf) - FfiConverterFloat.write(value.y, into: &buf) - FfiConverterFloat.write(value.height, into: &buf) - FfiConverterTypeRowKind.write(value.kind, into: &buf) + public static func write(_ value: StyleDesc, into buf: inout [UInt8]) { + FfiConverterUInt16.write(value.id, into: &buf) + FfiConverterTypeFaceRole.write(value.face, into: &buf) + FfiConverterFloat.write(value.size, into: &buf) + FfiConverterBool.write(value.ligatures, into: &buf) } } @@ -8161,38 +9361,35 @@ public struct FfiConverterTypeRowPlacement: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRowPlacement_lift(_ buf: RustBuffer) throws -> RowPlacement { - return try FfiConverterTypeRowPlacement.lift(buf) +public func FfiConverterTypeStyleDesc_lift(_ buf: RustBuffer) throws -> StyleDesc { + return try FfiConverterTypeStyleDesc.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeRowPlacement_lower(_ value: RowPlacement) -> RustBuffer { - return FfiConverterTypeRowPlacement.lower(value) +public func FfiConverterTypeStyleDesc_lower(_ value: StyleDesc) -> RustBuffer { + return FfiConverterTypeStyleDesc.lower(value) } /** - * A horizontally scrolling viewport (code blocks, wide tables). Primitives - * that reference it are positioned in its content coordinates, whose origin - * is the viewport's top-left. + * A surface's latest picture: an image (scenes, shaders) under vector + * paint (canvas drawings, charts, scene labels), in surface coordinates. */ -public struct Scroller: Equatable, Hashable { - public var x: Float - public var y: Float - public var w: Float - public var h: Float - public var contentWidth: Float +public struct SurfaceFrame: Equatable, Hashable { + public var surface: UInt64 + public var image: FrameImage? + public var paint: VisualPaint + public var styles: [StyleDesc] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(x: Float, y: Float, w: Float, h: Float, contentWidth: Float) { - self.x = x - self.y = y - self.w = w - self.h = h - self.contentWidth = contentWidth + public init(surface: UInt64, image: FrameImage?, paint: VisualPaint, styles: [StyleDesc]) { + self.surface = surface + self.image = image + self.paint = paint + self.styles = styles } @@ -8201,30 +9398,28 @@ public struct Scroller: Equatable, Hashable { } #if compiler(>=6) -extension Scroller: Sendable {} +extension SurfaceFrame: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeScroller: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Scroller { +public struct FfiConverterTypeSurfaceFrame: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SurfaceFrame { return - try Scroller( - x: FfiConverterFloat.read(from: &buf), - y: FfiConverterFloat.read(from: &buf), - w: FfiConverterFloat.read(from: &buf), - h: FfiConverterFloat.read(from: &buf), - contentWidth: FfiConverterFloat.read(from: &buf) + try SurfaceFrame( + surface: FfiConverterUInt64.read(from: &buf), + image: FfiConverterOptionTypeFrameImage.read(from: &buf), + paint: FfiConverterTypeVisualPaint.read(from: &buf), + styles: FfiConverterSequenceTypeStyleDesc.read(from: &buf) ) } - public static func write(_ value: Scroller, into buf: inout [UInt8]) { - FfiConverterFloat.write(value.x, into: &buf) - FfiConverterFloat.write(value.y, into: &buf) - FfiConverterFloat.write(value.w, into: &buf) - FfiConverterFloat.write(value.h, into: &buf) - FfiConverterFloat.write(value.contentWidth, into: &buf) + public static func write(_ value: SurfaceFrame, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.surface, into: &buf) + FfiConverterOptionTypeFrameImage.write(value.image, into: &buf) + FfiConverterTypeVisualPaint.write(value.paint, into: &buf) + FfiConverterSequenceTypeStyleDesc.write(value.styles, into: &buf) } } @@ -8232,35 +9427,67 @@ public struct FfiConverterTypeScroller: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeScroller_lift(_ buf: RustBuffer) throws -> Scroller { - return try FfiConverterTypeScroller.lift(buf) +public func FfiConverterTypeSurfaceFrame_lift(_ buf: RustBuffer) throws -> SurfaceFrame { + return try FfiConverterTypeSurfaceFrame.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeScroller_lower(_ value: Scroller) -> RustBuffer { - return FfiConverterTypeScroller.lower(value) +public func FfiConverterTypeSurfaceFrame_lower(_ value: SurfaceFrame) -> RustBuffer { + return FfiConverterTypeSurfaceFrame.lower(value) } -public struct SearchHit: Equatable, Hashable { - public var session: SessionRow - public var score: UInt32 +/** + * One positioned piece of text: `text[start..start+len]` (UTF-16) drawn with + * `style` so its origin sits at (`x`, `baseline`). + */ +public struct TextRun: Equatable, Hashable { + public var start: UInt32 + public var len: UInt32 + public var x: Float + public var baseline: Float /** - * The strongest field that matched. + * Rust-measured advance — painters may use it for selection/hit rects. */ - public var field: SearchField + public var width: Float + public var style: UInt16 + public var color: ColorRole + public var decoration: Decoration + /** + * Index into [`RowDisplay::scrollers`] when the run scrolls horizontally. + */ + public var scroller: UInt32? + /** + * Fade-in start in ms since the row's layout epoch (streaming veil); + * `None` = fully visible. + */ + public var revealMs: UInt32? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(session: SessionRow, score: UInt32, + public init(start: UInt32, len: UInt32, x: Float, baseline: Float, /** - * The strongest field that matched. - */field: SearchField) { - self.session = session - self.score = score - self.field = field + * Rust-measured advance — painters may use it for selection/hit rects. + */width: Float, style: UInt16, color: ColorRole, decoration: Decoration, + /** + * Index into [`RowDisplay::scrollers`] when the run scrolls horizontally. + */scroller: UInt32?, + /** + * Fade-in start in ms since the row's layout epoch (streaming veil); + * `None` = fully visible. + */revealMs: UInt32?) { + self.start = start + self.len = len + self.x = x + self.baseline = baseline + self.width = width + self.style = style + self.color = color + self.decoration = decoration + self.scroller = scroller + self.revealMs = revealMs } @@ -8269,26 +9496,40 @@ public struct SearchHit: Equatable, Hashable { } #if compiler(>=6) -extension SearchHit: Sendable {} +extension TextRun: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeSearchHit: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SearchHit { +public struct FfiConverterTypeTextRun: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> TextRun { return - try SearchHit( - session: FfiConverterTypeSessionRow.read(from: &buf), - score: FfiConverterUInt32.read(from: &buf), - field: FfiConverterTypeSearchField.read(from: &buf) + try TextRun( + start: FfiConverterUInt32.read(from: &buf), + len: FfiConverterUInt32.read(from: &buf), + x: FfiConverterFloat.read(from: &buf), + baseline: FfiConverterFloat.read(from: &buf), + width: FfiConverterFloat.read(from: &buf), + style: FfiConverterUInt16.read(from: &buf), + color: FfiConverterTypeColorRole.read(from: &buf), + decoration: FfiConverterTypeDecoration.read(from: &buf), + scroller: FfiConverterOptionUInt32.read(from: &buf), + revealMs: FfiConverterOptionUInt32.read(from: &buf) ) } - public static func write(_ value: SearchHit, into buf: inout [UInt8]) { - FfiConverterTypeSessionRow.write(value.session, into: &buf) - FfiConverterUInt32.write(value.score, into: &buf) - FfiConverterTypeSearchField.write(value.field, into: &buf) + public static func write(_ value: TextRun, into buf: inout [UInt8]) { + FfiConverterUInt32.write(value.start, into: &buf) + FfiConverterUInt32.write(value.len, into: &buf) + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.baseline, into: &buf) + FfiConverterFloat.write(value.width, into: &buf) + FfiConverterUInt16.write(value.style, into: &buf) + FfiConverterTypeColorRole.write(value.color, into: &buf) + FfiConverterTypeDecoration.write(value.decoration, into: &buf) + FfiConverterOptionUInt32.write(value.scroller, into: &buf) + FfiConverterOptionUInt32.write(value.revealMs, into: &buf) } } @@ -8296,37 +9537,61 @@ public struct FfiConverterTypeSearchHit: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeSearchHit_lift(_ buf: RustBuffer) throws -> SearchHit { - return try FfiConverterTypeSearchHit.lift(buf) +public func FfiConverterTypeTextRun_lift(_ buf: RustBuffer) throws -> TextRun { + return try FfiConverterTypeTextRun.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeSearchHit_lower(_ value: SearchHit) -> RustBuffer { - return FfiConverterTypeSearchHit.lower(value) +public func FfiConverterTypeTextRun_lower(_ value: TextRun) -> RustBuffer { + return FfiConverterTypeTextRun.lower(value) } -public struct SectionView: Equatable, Hashable { - public var id: String - public var name: String - public var collapsed: Bool +/** + * Transcript-level facts the platform needs without the rows (loader vs + * empty state, scroll-to-bottom affordances, the working pill). + */ +public struct TranscriptStatus: Equatable, Hashable { + public var revision: UInt64 /** - * Recency order (pinned sessions excluded). + * Content present (false = show a loader, not an empty chat). */ - public var sessions: [SessionRow] + public var hydrated: Bool + /** + * Host-written entries. + */ + public var entryCount: UInt32 + public var pendingCount: UInt32 + public var streaming: Bool + public var working: Bool + public var workingSinceMs: Int64? + /** + * Id of the newest entry (host or echo). + */ + public var lastEntryId: String? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, name: String, collapsed: Bool, + public init(revision: UInt64, /** - * Recency order (pinned sessions excluded). - */sessions: [SessionRow]) { - self.id = id - self.name = name - self.collapsed = collapsed - self.sessions = sessions + * Content present (false = show a loader, not an empty chat). + */hydrated: Bool, + /** + * Host-written entries. + */entryCount: UInt32, pendingCount: UInt32, streaming: Bool, working: Bool, workingSinceMs: Int64?, + /** + * Id of the newest entry (host or echo). + */lastEntryId: String?) { + self.revision = revision + self.hydrated = hydrated + self.entryCount = entryCount + self.pendingCount = pendingCount + self.streaming = streaming + self.working = working + self.workingSinceMs = workingSinceMs + self.lastEntryId = lastEntryId } @@ -8335,28 +9600,36 @@ public struct SectionView: Equatable, Hashable { } #if compiler(>=6) -extension SectionView: Sendable {} +extension TranscriptStatus: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeSectionView: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SectionView { +public struct FfiConverterTypeTranscriptStatus: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> TranscriptStatus { return - try SectionView( - id: FfiConverterString.read(from: &buf), - name: FfiConverterString.read(from: &buf), - collapsed: FfiConverterBool.read(from: &buf), - sessions: FfiConverterSequenceTypeSessionRow.read(from: &buf) + try TranscriptStatus( + revision: FfiConverterUInt64.read(from: &buf), + hydrated: FfiConverterBool.read(from: &buf), + entryCount: FfiConverterUInt32.read(from: &buf), + pendingCount: FfiConverterUInt32.read(from: &buf), + streaming: FfiConverterBool.read(from: &buf), + working: FfiConverterBool.read(from: &buf), + workingSinceMs: FfiConverterOptionInt64.read(from: &buf), + lastEntryId: FfiConverterOptionString.read(from: &buf) ) } - public static func write(_ value: SectionView, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.name, into: &buf) - FfiConverterBool.write(value.collapsed, into: &buf) - FfiConverterSequenceTypeSessionRow.write(value.sessions, into: &buf) + public static func write(_ value: TranscriptStatus, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.revision, into: &buf) + FfiConverterBool.write(value.hydrated, into: &buf) + FfiConverterUInt32.write(value.entryCount, into: &buf) + FfiConverterUInt32.write(value.pendingCount, into: &buf) + FfiConverterBool.write(value.streaming, into: &buf) + FfiConverterBool.write(value.working, into: &buf) + FfiConverterOptionInt64.write(value.workingSinceMs, into: &buf) + FfiConverterOptionString.write(value.lastEntryId, into: &buf) } } @@ -8364,37 +9637,35 @@ public struct FfiConverterTypeSectionView: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeSectionView_lift(_ buf: RustBuffer) throws -> SectionView { - return try FfiConverterTypeSectionView.lift(buf) +public func FfiConverterTypeTranscriptStatus_lift(_ buf: RustBuffer) throws -> TranscriptStatus { + return try FfiConverterTypeTranscriptStatus.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeSectionView_lower(_ value: SectionView) -> RustBuffer { - return FfiConverterTypeSectionView.lower(value) +public func FfiConverterTypeTranscriptStatus_lower(_ value: TranscriptStatus) -> RustBuffer { + return FfiConverterTypeTranscriptStatus.lower(value) } -public struct SendRequest: Equatable, Hashable { - public var text: String - public var attachments: [OutgoingAttachment] +public struct UserImage: Equatable, Hashable { /** - * Isolated host worktree for this run (new-session "New worktree"). + * Host path (or `pending://` ref) — read via `read_attachment`. */ - public var worktree: WorktreeSpec? - public var busy: BusyPolicy + public var path: String + public var name: String + public var appshot: AppshotLabel? // Default memberwise initializers are never public by default, so we // declare one manually. - public init(text: String, attachments: [OutgoingAttachment], + public init( /** - * Isolated host worktree for this run (new-session "New worktree"). - */worktree: WorktreeSpec?, busy: BusyPolicy) { - self.text = text - self.attachments = attachments - self.worktree = worktree - self.busy = busy + * Host path (or `pending://` ref) — read via `read_attachment`. + */path: String, name: String, appshot: AppshotLabel?) { + self.path = path + self.name = name + self.appshot = appshot } @@ -8403,181 +9674,60 @@ public struct SendRequest: Equatable, Hashable { } #if compiler(>=6) -extension SendRequest: Sendable {} +extension UserImage: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeSendRequest: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SendRequest { +public struct FfiConverterTypeUserImage: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UserImage { return - try SendRequest( - text: FfiConverterString.read(from: &buf), - attachments: FfiConverterSequenceTypeOutgoingAttachment.read(from: &buf), - worktree: FfiConverterOptionTypeWorktreeSpec.read(from: &buf), - busy: FfiConverterTypeBusyPolicy.read(from: &buf) + try UserImage( + path: FfiConverterString.read(from: &buf), + name: FfiConverterString.read(from: &buf), + appshot: FfiConverterOptionTypeAppshotLabel.read(from: &buf) ) } - public static func write(_ value: SendRequest, into buf: inout [UInt8]) { - FfiConverterString.write(value.text, into: &buf) - FfiConverterSequenceTypeOutgoingAttachment.write(value.attachments, into: &buf) - FfiConverterOptionTypeWorktreeSpec.write(value.worktree, into: &buf) - FfiConverterTypeBusyPolicy.write(value.busy, into: &buf) + public static func write(_ value: UserImage, into buf: inout [UInt8]) { + FfiConverterString.write(value.path, into: &buf) + FfiConverterString.write(value.name, into: &buf) + FfiConverterOptionTypeAppshotLabel.write(value.appshot, into: &buf) } } #if swift(>=5.8) @_documentation(visibility: private) -#endif -public func FfiConverterTypeSendRequest_lift(_ buf: RustBuffer) throws -> SendRequest { - return try FfiConverterTypeSendRequest.lift(buf) -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeSendRequest_lower(_ value: SendRequest) -> RustBuffer { - return FfiConverterTypeSendRequest.lower(value) -} - - -/** - * One session row, as every list renders it. - */ -public struct SessionRow: Equatable, Hashable { - public var id: String - /** - * Content hash — equal ⇒ nothing to redraw. - */ - public var revision: UInt64 - /** - * Display title ("New session" when untitled). - */ - public var title: String - public var hasTitle: Bool - public var preview: String? - /** - * `None` = projectless (host's home folder). - */ - public var project: ProjectRef? - public var deviceId: String - public var deviceName: String? - public var deviceOnline: Bool - /** - * Wire harness id (`claude-code`). - */ - public var harness: String? - public var harnessLabel: String? - public var model: String? - public var modelLabel: String? - /** - * Wire effort (`xhigh`). - */ - public var reasoning: String? - public var branch: String? - public var cwd: String? - /** - * Staleness-gated (45s) live status; an in-flight send reads Working. - */ - public var indicator: ChatIndicator - /** - * Host-reported status only (no local-send override). - */ - public var hostIndicator: ChatIndicator - /** - * Run start of the live turn while Working/AwaitingInput. - */ - public var workingSinceMs: Int64? - public var lastActivityMs: Int64 - /** - * `now` / `34m` / `4h` / `2d` at derive time. - */ - public var timeLabel: String - public var createdAtMs: Int64 - public var unseen: Bool - public var archived: Bool - public var pinned: Bool - public var sectionId: String? - public var pullRequest: PullRequest? - /** - * Oldest unadopted send from this device (open sessions only). - */ - public var sendState: SendState? - public var parentChatId: String? +#endif +public func FfiConverterTypeUserImage_lift(_ buf: RustBuffer) throws -> UserImage { + return try FfiConverterTypeUserImage.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeUserImage_lower(_ value: UserImage) -> RustBuffer { + return FfiConverterTypeUserImage.lower(value) +} + + +public struct UserInputAnswer: Equatable, Hashable { + public var questionId: String /** - * 2 = chat2; 1 = legacy (not dialable). + * Picked option labels (or free text as a single label). */ - public var roomGen: UInt32 + public var labels: [String] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, - /** - * Content hash — equal ⇒ nothing to redraw. - */revision: UInt64, - /** - * Display title ("New session" when untitled). - */title: String, hasTitle: Bool, preview: String?, - /** - * `None` = projectless (host's home folder). - */project: ProjectRef?, deviceId: String, deviceName: String?, deviceOnline: Bool, - /** - * Wire harness id (`claude-code`). - */harness: String?, harnessLabel: String?, model: String?, modelLabel: String?, - /** - * Wire effort (`xhigh`). - */reasoning: String?, branch: String?, cwd: String?, - /** - * Staleness-gated (45s) live status; an in-flight send reads Working. - */indicator: ChatIndicator, - /** - * Host-reported status only (no local-send override). - */hostIndicator: ChatIndicator, - /** - * Run start of the live turn while Working/AwaitingInput. - */workingSinceMs: Int64?, lastActivityMs: Int64, - /** - * `now` / `34m` / `4h` / `2d` at derive time. - */timeLabel: String, createdAtMs: Int64, unseen: Bool, archived: Bool, pinned: Bool, sectionId: String?, pullRequest: PullRequest?, - /** - * Oldest unadopted send from this device (open sessions only). - */sendState: SendState?, parentChatId: String?, + public init(questionId: String, /** - * 2 = chat2; 1 = legacy (not dialable). - */roomGen: UInt32) { - self.id = id - self.revision = revision - self.title = title - self.hasTitle = hasTitle - self.preview = preview - self.project = project - self.deviceId = deviceId - self.deviceName = deviceName - self.deviceOnline = deviceOnline - self.harness = harness - self.harnessLabel = harnessLabel - self.model = model - self.modelLabel = modelLabel - self.reasoning = reasoning - self.branch = branch - self.cwd = cwd - self.indicator = indicator - self.hostIndicator = hostIndicator - self.workingSinceMs = workingSinceMs - self.lastActivityMs = lastActivityMs - self.timeLabel = timeLabel - self.createdAtMs = createdAtMs - self.unseen = unseen - self.archived = archived - self.pinned = pinned - self.sectionId = sectionId - self.pullRequest = pullRequest - self.sendState = sendState - self.parentChatId = parentChatId - self.roomGen = roomGen + * Picked option labels (or free text as a single label). + */labels: [String]) { + self.questionId = questionId + self.labels = labels } @@ -8586,80 +9736,24 @@ public struct SessionRow: Equatable, Hashable { } #if compiler(>=6) -extension SessionRow: Sendable {} +extension UserInputAnswer: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeSessionRow: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SessionRow { +public struct FfiConverterTypeUserInputAnswer: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UserInputAnswer { return - try SessionRow( - id: FfiConverterString.read(from: &buf), - revision: FfiConverterUInt64.read(from: &buf), - title: FfiConverterString.read(from: &buf), - hasTitle: FfiConverterBool.read(from: &buf), - preview: FfiConverterOptionString.read(from: &buf), - project: FfiConverterOptionTypeProjectRef.read(from: &buf), - deviceId: FfiConverterString.read(from: &buf), - deviceName: FfiConverterOptionString.read(from: &buf), - deviceOnline: FfiConverterBool.read(from: &buf), - harness: FfiConverterOptionString.read(from: &buf), - harnessLabel: FfiConverterOptionString.read(from: &buf), - model: FfiConverterOptionString.read(from: &buf), - modelLabel: FfiConverterOptionString.read(from: &buf), - reasoning: FfiConverterOptionString.read(from: &buf), - branch: FfiConverterOptionString.read(from: &buf), - cwd: FfiConverterOptionString.read(from: &buf), - indicator: FfiConverterTypeChatIndicator.read(from: &buf), - hostIndicator: FfiConverterTypeChatIndicator.read(from: &buf), - workingSinceMs: FfiConverterOptionInt64.read(from: &buf), - lastActivityMs: FfiConverterInt64.read(from: &buf), - timeLabel: FfiConverterString.read(from: &buf), - createdAtMs: FfiConverterInt64.read(from: &buf), - unseen: FfiConverterBool.read(from: &buf), - archived: FfiConverterBool.read(from: &buf), - pinned: FfiConverterBool.read(from: &buf), - sectionId: FfiConverterOptionString.read(from: &buf), - pullRequest: FfiConverterOptionTypePullRequest.read(from: &buf), - sendState: FfiConverterOptionTypeSendState.read(from: &buf), - parentChatId: FfiConverterOptionString.read(from: &buf), - roomGen: FfiConverterUInt32.read(from: &buf) + try UserInputAnswer( + questionId: FfiConverterString.read(from: &buf), + labels: FfiConverterSequenceString.read(from: &buf) ) } - public static func write(_ value: SessionRow, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterUInt64.write(value.revision, into: &buf) - FfiConverterString.write(value.title, into: &buf) - FfiConverterBool.write(value.hasTitle, into: &buf) - FfiConverterOptionString.write(value.preview, into: &buf) - FfiConverterOptionTypeProjectRef.write(value.project, into: &buf) - FfiConverterString.write(value.deviceId, into: &buf) - FfiConverterOptionString.write(value.deviceName, into: &buf) - FfiConverterBool.write(value.deviceOnline, into: &buf) - FfiConverterOptionString.write(value.harness, into: &buf) - FfiConverterOptionString.write(value.harnessLabel, into: &buf) - FfiConverterOptionString.write(value.model, into: &buf) - FfiConverterOptionString.write(value.modelLabel, into: &buf) - FfiConverterOptionString.write(value.reasoning, into: &buf) - FfiConverterOptionString.write(value.branch, into: &buf) - FfiConverterOptionString.write(value.cwd, into: &buf) - FfiConverterTypeChatIndicator.write(value.indicator, into: &buf) - FfiConverterTypeChatIndicator.write(value.hostIndicator, into: &buf) - FfiConverterOptionInt64.write(value.workingSinceMs, into: &buf) - FfiConverterInt64.write(value.lastActivityMs, into: &buf) - FfiConverterString.write(value.timeLabel, into: &buf) - FfiConverterInt64.write(value.createdAtMs, into: &buf) - FfiConverterBool.write(value.unseen, into: &buf) - FfiConverterBool.write(value.archived, into: &buf) - FfiConverterBool.write(value.pinned, into: &buf) - FfiConverterOptionString.write(value.sectionId, into: &buf) - FfiConverterOptionTypePullRequest.write(value.pullRequest, into: &buf) - FfiConverterOptionTypeSendState.write(value.sendState, into: &buf) - FfiConverterOptionString.write(value.parentChatId, into: &buf) - FfiConverterUInt32.write(value.roomGen, into: &buf) + public static func write(_ value: UserInputAnswer, into buf: inout [UInt8]) { + FfiConverterString.write(value.questionId, into: &buf) + FfiConverterSequenceString.write(value.labels, into: &buf) } } @@ -8667,34 +9761,37 @@ public struct FfiConverterTypeSessionRow: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeSessionRow_lift(_ buf: RustBuffer) throws -> SessionRow { - return try FfiConverterTypeSessionRow.lift(buf) +public func FfiConverterTypeUserInputAnswer_lift(_ buf: RustBuffer) throws -> UserInputAnswer { + return try FfiConverterTypeUserInputAnswer.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeSessionRow_lower(_ value: SessionRow) -> RustBuffer { - return FfiConverterTypeSessionRow.lower(value) +public func FfiConverterTypeUserInputAnswer_lower(_ value: UserInputAnswer) -> RustBuffer { + return FfiConverterTypeUserInputAnswer.lower(value) } -/** - * A style the platform must be able to draw: `id` → (face, size). - */ -public struct StyleDesc: Equatable, Hashable { - public var id: UInt16 - public var face: FaceRole - public var size: Float - public var ligatures: Bool +public struct UserInputQuestion: Equatable, Hashable { + public var id: String + public var header: String + public var question: String + public var options: [String] + public var multiSelect: Bool + public var prefill: String? + public var multiline: Bool // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: UInt16, face: FaceRole, size: Float, ligatures: Bool) { + public init(id: String, header: String, question: String, options: [String], multiSelect: Bool, prefill: String?, multiline: Bool) { self.id = id - self.face = face - self.size = size - self.ligatures = ligatures + self.header = header + self.question = question + self.options = options + self.multiSelect = multiSelect + self.prefill = prefill + self.multiline = multiline } @@ -8703,28 +9800,34 @@ public struct StyleDesc: Equatable, Hashable { } #if compiler(>=6) -extension StyleDesc: Sendable {} +extension UserInputQuestion: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeStyleDesc: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> StyleDesc { +public struct FfiConverterTypeUserInputQuestion: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UserInputQuestion { return - try StyleDesc( - id: FfiConverterUInt16.read(from: &buf), - face: FfiConverterTypeFaceRole.read(from: &buf), - size: FfiConverterFloat.read(from: &buf), - ligatures: FfiConverterBool.read(from: &buf) + try UserInputQuestion( + id: FfiConverterString.read(from: &buf), + header: FfiConverterString.read(from: &buf), + question: FfiConverterString.read(from: &buf), + options: FfiConverterSequenceString.read(from: &buf), + multiSelect: FfiConverterBool.read(from: &buf), + prefill: FfiConverterOptionString.read(from: &buf), + multiline: FfiConverterBool.read(from: &buf) ) } - public static func write(_ value: StyleDesc, into buf: inout [UInt8]) { - FfiConverterUInt16.write(value.id, into: &buf) - FfiConverterTypeFaceRole.write(value.face, into: &buf) - FfiConverterFloat.write(value.size, into: &buf) - FfiConverterBool.write(value.ligatures, into: &buf) + public static func write(_ value: UserInputQuestion, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterString.write(value.header, into: &buf) + FfiConverterString.write(value.question, into: &buf) + FfiConverterSequenceString.write(value.options, into: &buf) + FfiConverterBool.write(value.multiSelect, into: &buf) + FfiConverterOptionString.write(value.prefill, into: &buf) + FfiConverterBool.write(value.multiline, into: &buf) } } @@ -8732,67 +9835,52 @@ public struct FfiConverterTypeStyleDesc: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeStyleDesc_lift(_ buf: RustBuffer) throws -> StyleDesc { - return try FfiConverterTypeStyleDesc.lift(buf) -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeStyleDesc_lower(_ value: StyleDesc) -> RustBuffer { - return FfiConverterTypeStyleDesc.lower(value) +public func FfiConverterTypeUserInputQuestion_lift(_ buf: RustBuffer) throws -> UserInputQuestion { + return try FfiConverterTypeUserInputQuestion.lift(buf) } - -/** - * One positioned piece of text: `text[start..start+len]` (UTF-16) drawn with - * `style` so its origin sits at (`x`, `baseline`). - */ -public struct TextRun: Equatable, Hashable { - public var start: UInt32 - public var len: UInt32 - public var x: Float - public var baseline: Float - /** - * Rust-measured advance — painters may use it for selection/hit rects. - */ - public var width: Float - public var style: UInt16 - public var color: ColorRole - public var decoration: Decoration +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeUserInputQuestion_lower(_ value: UserInputQuestion) -> RustBuffer { + return FfiConverterTypeUserInputQuestion.lower(value) +} + + +/** + * A touch target over painted control art. + */ +public struct VisualControl: Equatable, Hashable { /** - * Index into [`RowDisplay::scrollers`] when the run scrolls horizontally. + * Stable across renders. */ - public var scroller: UInt32? + public var id: UInt64 + public var kind: ControlKind + public var x: Float + public var y: Float + public var w: Float + public var h: Float /** - * Fade-in start in ms since the row's layout epoch (streaming veil); - * `None` = fully visible. + * Accessibility label. */ - public var revealMs: UInt32? + public var label: String // Default memberwise initializers are never public by default, so we // declare one manually. - public init(start: UInt32, len: UInt32, x: Float, baseline: Float, - /** - * Rust-measured advance — painters may use it for selection/hit rects. - */width: Float, style: UInt16, color: ColorRole, decoration: Decoration, + public init( /** - * Index into [`RowDisplay::scrollers`] when the run scrolls horizontally. - */scroller: UInt32?, + * Stable across renders. + */id: UInt64, kind: ControlKind, x: Float, y: Float, w: Float, h: Float, /** - * Fade-in start in ms since the row's layout epoch (streaming veil); - * `None` = fully visible. - */revealMs: UInt32?) { - self.start = start - self.len = len + * Accessibility label. + */label: String) { + self.id = id + self.kind = kind self.x = x - self.baseline = baseline - self.width = width - self.style = style - self.color = color - self.decoration = decoration - self.scroller = scroller - self.revealMs = revealMs + self.y = y + self.w = w + self.h = h + self.label = label } @@ -8801,40 +9889,34 @@ public struct TextRun: Equatable, Hashable { } #if compiler(>=6) -extension TextRun: Sendable {} +extension VisualControl: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeTextRun: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> TextRun { +public struct FfiConverterTypeVisualControl: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualControl { return - try TextRun( - start: FfiConverterUInt32.read(from: &buf), - len: FfiConverterUInt32.read(from: &buf), + try VisualControl( + id: FfiConverterUInt64.read(from: &buf), + kind: FfiConverterTypeControlKind.read(from: &buf), x: FfiConverterFloat.read(from: &buf), - baseline: FfiConverterFloat.read(from: &buf), - width: FfiConverterFloat.read(from: &buf), - style: FfiConverterUInt16.read(from: &buf), - color: FfiConverterTypeColorRole.read(from: &buf), - decoration: FfiConverterTypeDecoration.read(from: &buf), - scroller: FfiConverterOptionUInt32.read(from: &buf), - revealMs: FfiConverterOptionUInt32.read(from: &buf) + y: FfiConverterFloat.read(from: &buf), + w: FfiConverterFloat.read(from: &buf), + h: FfiConverterFloat.read(from: &buf), + label: FfiConverterString.read(from: &buf) ) } - public static func write(_ value: TextRun, into buf: inout [UInt8]) { - FfiConverterUInt32.write(value.start, into: &buf) - FfiConverterUInt32.write(value.len, into: &buf) + public static func write(_ value: VisualControl, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.id, into: &buf) + FfiConverterTypeControlKind.write(value.kind, into: &buf) FfiConverterFloat.write(value.x, into: &buf) - FfiConverterFloat.write(value.baseline, into: &buf) - FfiConverterFloat.write(value.width, into: &buf) - FfiConverterUInt16.write(value.style, into: &buf) - FfiConverterTypeColorRole.write(value.color, into: &buf) - FfiConverterTypeDecoration.write(value.decoration, into: &buf) - FfiConverterOptionUInt32.write(value.scroller, into: &buf) - FfiConverterOptionUInt32.write(value.revealMs, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.w, into: &buf) + FfiConverterFloat.write(value.h, into: &buf) + FfiConverterString.write(value.label, into: &buf) } } @@ -8842,61 +9924,48 @@ public struct FfiConverterTypeTextRun: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeTextRun_lift(_ buf: RustBuffer) throws -> TextRun { - return try FfiConverterTypeTextRun.lift(buf) +public func FfiConverterTypeVisualControl_lift(_ buf: RustBuffer) throws -> VisualControl { + return try FfiConverterTypeVisualControl.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeTextRun_lower(_ value: TextRun) -> RustBuffer { - return FfiConverterTypeTextRun.lower(value) +public func FfiConverterTypeVisualControl_lower(_ value: VisualControl) -> RustBuffer { + return FfiConverterTypeVisualControl.lower(value) } /** - * Transcript-level facts the platform needs without the rows (loader vs - * empty state, scroll-to-bottom affordances, the working pill). + * A visual laid out at one width. */ -public struct TranscriptStatus: Equatable, Hashable { +public struct VisualDisplay: Equatable, Hashable { public var revision: UInt64 + public var width: Float + public var height: Float + public var dark: Bool + public var paint: VisualPaint + public var surfaces: [VisualSurface] + public var controls: [VisualControl] /** - * Content present (false = show a loader, not an empty chat). - */ - public var hydrated: Bool - /** - * Host-written entries. - */ - public var entryCount: UInt32 - public var pendingCount: UInt32 - public var streaming: Bool - public var working: Bool - public var workingSinceMs: Int64? - /** - * Id of the newest entry (host or echo). + * Style id → face and size, for every run so far. */ - public var lastEntryId: String? + public var styles: [StyleDesc] // Default memberwise initializers are never public by default, so we // declare one manually. - public init(revision: UInt64, - /** - * Content present (false = show a loader, not an empty chat). - */hydrated: Bool, - /** - * Host-written entries. - */entryCount: UInt32, pendingCount: UInt32, streaming: Bool, working: Bool, workingSinceMs: Int64?, + public init(revision: UInt64, width: Float, height: Float, dark: Bool, paint: VisualPaint, surfaces: [VisualSurface], controls: [VisualControl], /** - * Id of the newest entry (host or echo). - */lastEntryId: String?) { + * Style id → face and size, for every run so far. + */styles: [StyleDesc]) { self.revision = revision - self.hydrated = hydrated - self.entryCount = entryCount - self.pendingCount = pendingCount - self.streaming = streaming - self.working = working - self.workingSinceMs = workingSinceMs - self.lastEntryId = lastEntryId + self.width = width + self.height = height + self.dark = dark + self.paint = paint + self.surfaces = surfaces + self.controls = controls + self.styles = styles } @@ -8905,36 +9974,36 @@ public struct TranscriptStatus: Equatable, Hashable { } #if compiler(>=6) -extension TranscriptStatus: Sendable {} +extension VisualDisplay: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeTranscriptStatus: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> TranscriptStatus { +public struct FfiConverterTypeVisualDisplay: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualDisplay { return - try TranscriptStatus( + try VisualDisplay( revision: FfiConverterUInt64.read(from: &buf), - hydrated: FfiConverterBool.read(from: &buf), - entryCount: FfiConverterUInt32.read(from: &buf), - pendingCount: FfiConverterUInt32.read(from: &buf), - streaming: FfiConverterBool.read(from: &buf), - working: FfiConverterBool.read(from: &buf), - workingSinceMs: FfiConverterOptionInt64.read(from: &buf), - lastEntryId: FfiConverterOptionString.read(from: &buf) + width: FfiConverterFloat.read(from: &buf), + height: FfiConverterFloat.read(from: &buf), + dark: FfiConverterBool.read(from: &buf), + paint: FfiConverterTypeVisualPaint.read(from: &buf), + surfaces: FfiConverterSequenceTypeVisualSurface.read(from: &buf), + controls: FfiConverterSequenceTypeVisualControl.read(from: &buf), + styles: FfiConverterSequenceTypeStyleDesc.read(from: &buf) ) } - public static func write(_ value: TranscriptStatus, into buf: inout [UInt8]) { + public static func write(_ value: VisualDisplay, into buf: inout [UInt8]) { FfiConverterUInt64.write(value.revision, into: &buf) - FfiConverterBool.write(value.hydrated, into: &buf) - FfiConverterUInt32.write(value.entryCount, into: &buf) - FfiConverterUInt32.write(value.pendingCount, into: &buf) - FfiConverterBool.write(value.streaming, into: &buf) - FfiConverterBool.write(value.working, into: &buf) - FfiConverterOptionInt64.write(value.workingSinceMs, into: &buf) - FfiConverterOptionString.write(value.lastEntryId, into: &buf) + FfiConverterFloat.write(value.width, into: &buf) + FfiConverterFloat.write(value.height, into: &buf) + FfiConverterBool.write(value.dark, into: &buf) + FfiConverterTypeVisualPaint.write(value.paint, into: &buf) + FfiConverterSequenceTypeVisualSurface.write(value.surfaces, into: &buf) + FfiConverterSequenceTypeVisualControl.write(value.controls, into: &buf) + FfiConverterSequenceTypeStyleDesc.write(value.styles, into: &buf) } } @@ -8942,35 +10011,32 @@ public struct FfiConverterTypeTranscriptStatus: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeTranscriptStatus_lift(_ buf: RustBuffer) throws -> TranscriptStatus { - return try FfiConverterTypeTranscriptStatus.lift(buf) +public func FfiConverterTypeVisualDisplay_lift(_ buf: RustBuffer) throws -> VisualDisplay { + return try FfiConverterTypeVisualDisplay.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeTranscriptStatus_lower(_ value: TranscriptStatus) -> RustBuffer { - return FfiConverterTypeTranscriptStatus.lower(value) +public func FfiConverterTypeVisualDisplay_lower(_ value: VisualDisplay) -> RustBuffer { + return FfiConverterTypeVisualDisplay.lower(value) } -public struct UserImage: Equatable, Hashable { - /** - * Host path (or `pending://` ref) — read via `read_attachment`. - */ - public var path: String - public var name: String - public var appshot: AppshotLabel? +/** + * One layer of vector paint: shapes, then text over them. + */ +public struct VisualPaint: Equatable, Hashable { + public var text: String + public var runs: [VisualRun] + public var shapes: [VisualShape] // Default memberwise initializers are never public by default, so we // declare one manually. - public init( - /** - * Host path (or `pending://` ref) — read via `read_attachment`. - */path: String, name: String, appshot: AppshotLabel?) { - self.path = path - self.name = name - self.appshot = appshot + public init(text: String, runs: [VisualRun], shapes: [VisualShape]) { + self.text = text + self.runs = runs + self.shapes = shapes } @@ -8979,26 +10045,26 @@ public struct UserImage: Equatable, Hashable { } #if compiler(>=6) -extension UserImage: Sendable {} +extension VisualPaint: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeUserImage: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UserImage { +public struct FfiConverterTypeVisualPaint: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualPaint { return - try UserImage( - path: FfiConverterString.read(from: &buf), - name: FfiConverterString.read(from: &buf), - appshot: FfiConverterOptionTypeAppshotLabel.read(from: &buf) + try VisualPaint( + text: FfiConverterString.read(from: &buf), + runs: FfiConverterSequenceTypeVisualRun.read(from: &buf), + shapes: FfiConverterSequenceTypeVisualShape.read(from: &buf) ) } - public static func write(_ value: UserImage, into buf: inout [UInt8]) { - FfiConverterString.write(value.path, into: &buf) - FfiConverterString.write(value.name, into: &buf) - FfiConverterOptionTypeAppshotLabel.write(value.appshot, into: &buf) + public static func write(_ value: VisualPaint, into buf: inout [UInt8]) { + FfiConverterString.write(value.text, into: &buf) + FfiConverterSequenceTypeVisualRun.write(value.runs, into: &buf) + FfiConverterSequenceTypeVisualShape.write(value.shapes, into: &buf) } } @@ -9006,33 +10072,40 @@ public struct FfiConverterTypeUserImage: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeUserImage_lift(_ buf: RustBuffer) throws -> UserImage { - return try FfiConverterTypeUserImage.lift(buf) +public func FfiConverterTypeVisualPaint_lift(_ buf: RustBuffer) throws -> VisualPaint { + return try FfiConverterTypeVisualPaint.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeUserImage_lower(_ value: UserImage) -> RustBuffer { - return FfiConverterTypeUserImage.lower(value) +public func FfiConverterTypeVisualPaint_lower(_ value: VisualPaint) -> RustBuffer { + return FfiConverterTypeVisualPaint.lower(value) } -public struct UserInputAnswer: Equatable, Hashable { - public var questionId: String - /** - * Picked option labels (or free text as a single label). - */ - public var labels: [String] +/** + * `text[start..start+len]` (UTF-16) in `style`, its origin at (x, baseline). + */ +public struct VisualRun: Equatable, Hashable { + public var start: UInt32 + public var len: UInt32 + public var x: Float + public var baseline: Float + public var width: Float + public var style: UInt16 + public var color: Rgba // Default memberwise initializers are never public by default, so we // declare one manually. - public init(questionId: String, - /** - * Picked option labels (or free text as a single label). - */labels: [String]) { - self.questionId = questionId - self.labels = labels + public init(start: UInt32, len: UInt32, x: Float, baseline: Float, width: Float, style: UInt16, color: Rgba) { + self.start = start + self.len = len + self.x = x + self.baseline = baseline + self.width = width + self.style = style + self.color = color } @@ -9041,24 +10114,34 @@ public struct UserInputAnswer: Equatable, Hashable { } #if compiler(>=6) -extension UserInputAnswer: Sendable {} +extension VisualRun: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeUserInputAnswer: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UserInputAnswer { +public struct FfiConverterTypeVisualRun: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualRun { return - try UserInputAnswer( - questionId: FfiConverterString.read(from: &buf), - labels: FfiConverterSequenceString.read(from: &buf) + try VisualRun( + start: FfiConverterUInt32.read(from: &buf), + len: FfiConverterUInt32.read(from: &buf), + x: FfiConverterFloat.read(from: &buf), + baseline: FfiConverterFloat.read(from: &buf), + width: FfiConverterFloat.read(from: &buf), + style: FfiConverterUInt16.read(from: &buf), + color: FfiConverterTypeRgba.read(from: &buf) ) } - public static func write(_ value: UserInputAnswer, into buf: inout [UInt8]) { - FfiConverterString.write(value.questionId, into: &buf) - FfiConverterSequenceString.write(value.labels, into: &buf) + public static func write(_ value: VisualRun, into buf: inout [UInt8]) { + FfiConverterUInt32.write(value.start, into: &buf) + FfiConverterUInt32.write(value.len, into: &buf) + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.baseline, into: &buf) + FfiConverterFloat.write(value.width, into: &buf) + FfiConverterUInt16.write(value.style, into: &buf) + FfiConverterTypeRgba.write(value.color, into: &buf) } } @@ -9066,37 +10149,44 @@ public struct FfiConverterTypeUserInputAnswer: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeUserInputAnswer_lift(_ buf: RustBuffer) throws -> UserInputAnswer { - return try FfiConverterTypeUserInputAnswer.lift(buf) +public func FfiConverterTypeVisualRun_lift(_ buf: RustBuffer) throws -> VisualRun { + return try FfiConverterTypeVisualRun.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeUserInputAnswer_lower(_ value: UserInputAnswer) -> RustBuffer { - return FfiConverterTypeUserInputAnswer.lower(value) +public func FfiConverterTypeVisualRun_lower(_ value: VisualRun) -> RustBuffer { + return FfiConverterTypeVisualRun.lower(value) } -public struct UserInputQuestion: Equatable, Hashable { - public var id: String - public var header: String - public var question: String - public var options: [String] - public var multiSelect: Bool - public var prefill: String? - public var multiline: Bool +/** + * A region the worker redraws on its own (canvas, chart, scene, shader). + */ +public struct VisualSurface: Equatable, Hashable { + /** + * Stable across renders: frames and touches address it. + */ + public var id: UInt64 + public var x: Float + public var y: Float + public var w: Float + public var h: Float + public var gesture: SurfaceGesture // Default memberwise initializers are never public by default, so we // declare one manually. - public init(id: String, header: String, question: String, options: [String], multiSelect: Bool, prefill: String?, multiline: Bool) { - self.id = id - self.header = header - self.question = question - self.options = options - self.multiSelect = multiSelect - self.prefill = prefill - self.multiline = multiline + public init( + /** + * Stable across renders: frames and touches address it. + */id: UInt64, x: Float, y: Float, w: Float, h: Float, gesture: SurfaceGesture) { + self.id = id + self.x = x + self.y = y + self.w = w + self.h = h + self.gesture = gesture } @@ -9105,34 +10195,32 @@ public struct UserInputQuestion: Equatable, Hashable { } #if compiler(>=6) -extension UserInputQuestion: Sendable {} +extension VisualSurface: Sendable {} #endif #if swift(>=5.8) @_documentation(visibility: private) #endif -public struct FfiConverterTypeUserInputQuestion: FfiConverterRustBuffer { - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UserInputQuestion { +public struct FfiConverterTypeVisualSurface: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualSurface { return - try UserInputQuestion( - id: FfiConverterString.read(from: &buf), - header: FfiConverterString.read(from: &buf), - question: FfiConverterString.read(from: &buf), - options: FfiConverterSequenceString.read(from: &buf), - multiSelect: FfiConverterBool.read(from: &buf), - prefill: FfiConverterOptionString.read(from: &buf), - multiline: FfiConverterBool.read(from: &buf) + try VisualSurface( + id: FfiConverterUInt64.read(from: &buf), + x: FfiConverterFloat.read(from: &buf), + y: FfiConverterFloat.read(from: &buf), + w: FfiConverterFloat.read(from: &buf), + h: FfiConverterFloat.read(from: &buf), + gesture: FfiConverterTypeSurfaceGesture.read(from: &buf) ) } - public static func write(_ value: UserInputQuestion, into buf: inout [UInt8]) { - FfiConverterString.write(value.id, into: &buf) - FfiConverterString.write(value.header, into: &buf) - FfiConverterString.write(value.question, into: &buf) - FfiConverterSequenceString.write(value.options, into: &buf) - FfiConverterBool.write(value.multiSelect, into: &buf) - FfiConverterOptionString.write(value.prefill, into: &buf) - FfiConverterBool.write(value.multiline, into: &buf) + public static func write(_ value: VisualSurface, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.id, into: &buf) + FfiConverterFloat.write(value.x, into: &buf) + FfiConverterFloat.write(value.y, into: &buf) + FfiConverterFloat.write(value.w, into: &buf) + FfiConverterFloat.write(value.h, into: &buf) + FfiConverterTypeSurfaceGesture.write(value.gesture, into: &buf) } } @@ -9140,15 +10228,15 @@ public struct FfiConverterTypeUserInputQuestion: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeUserInputQuestion_lift(_ buf: RustBuffer) throws -> UserInputQuestion { - return try FfiConverterTypeUserInputQuestion.lift(buf) +public func FfiConverterTypeVisualSurface_lift(_ buf: RustBuffer) throws -> VisualSurface { + return try FfiConverterTypeVisualSurface.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeUserInputQuestion_lower(_ value: UserInputQuestion) -> RustBuffer { - return FfiConverterTypeUserInputQuestion.lower(value) +public func FfiConverterTypeVisualSurface_lower(_ value: VisualSurface) -> RustBuffer { + return FfiConverterTypeVisualSurface.lower(value) } @@ -10534,6 +11622,131 @@ public func FfiConverterTypeConnectivityState_lower(_ value: ConnectivityState) + +public enum ControlKind: Equatable, Hashable { + + case button(primary: Bool + ) + case tab(selected: Bool + ) + case toggle(on: Bool + ) + /** + * The track runs from `track_x` for `track_w` points. + */ + case slider(value: Double, min: Double, max: Double, step: Double, fraction: Float, trackX: Float, trackW: Float, shown: String + ) + /** + * Opened by the platform. + */ + case link(url: String + ) + /** + * A tappable stack. + */ + case tap + + + + + +} + +#if compiler(>=6) +extension ControlKind: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeControlKind: FfiConverterRustBuffer { + typealias SwiftType = ControlKind + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ControlKind { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .button(primary: try FfiConverterBool.read(from: &buf) + ) + + case 2: return .tab(selected: try FfiConverterBool.read(from: &buf) + ) + + case 3: return .toggle(on: try FfiConverterBool.read(from: &buf) + ) + + case 4: return .slider(value: try FfiConverterDouble.read(from: &buf), min: try FfiConverterDouble.read(from: &buf), max: try FfiConverterDouble.read(from: &buf), step: try FfiConverterDouble.read(from: &buf), fraction: try FfiConverterFloat.read(from: &buf), trackX: try FfiConverterFloat.read(from: &buf), trackW: try FfiConverterFloat.read(from: &buf), shown: try FfiConverterString.read(from: &buf) + ) + + case 5: return .link(url: try FfiConverterString.read(from: &buf) + ) + + case 6: return .tap + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: ControlKind, into buf: inout [UInt8]) { + switch value { + + + case let .button(primary): + writeInt(&buf, Int32(1)) + FfiConverterBool.write(primary, into: &buf) + + + case let .tab(selected): + writeInt(&buf, Int32(2)) + FfiConverterBool.write(selected, into: &buf) + + + case let .toggle(on): + writeInt(&buf, Int32(3)) + FfiConverterBool.write(on, into: &buf) + + + case let .slider(value,min,max,step,fraction,trackX,trackW,shown): + writeInt(&buf, Int32(4)) + FfiConverterDouble.write(value, into: &buf) + FfiConverterDouble.write(min, into: &buf) + FfiConverterDouble.write(max, into: &buf) + FfiConverterDouble.write(step, into: &buf) + FfiConverterFloat.write(fraction, into: &buf) + FfiConverterFloat.write(trackX, into: &buf) + FfiConverterFloat.write(trackW, into: &buf) + FfiConverterString.write(shown, into: &buf) + + + case let .link(url): + writeInt(&buf, Int32(5)) + FfiConverterString.write(url, into: &buf) + + + case .tap: + writeInt(&buf, Int32(6)) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeControlKind_lift(_ buf: RustBuffer) throws -> ControlKind { + return try FfiConverterTypeControlKind.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeControlKind_lower(_ value: ControlKind) -> RustBuffer { + return FfiConverterTypeControlKind.lower(value) +} + + + public enum CoreError: Swift.Error, Equatable, Hashable, Foundation.LocalizedError { @@ -11335,6 +12548,86 @@ public func FfiConverterTypePendingKind_lower(_ value: PendingKind) -> RustBuffe +public enum PointerPhase: Equatable, Hashable { + + case down + case move + case up + case cancel + + + + + +} + +#if compiler(>=6) +extension PointerPhase: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypePointerPhase: FfiConverterRustBuffer { + typealias SwiftType = PointerPhase + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PointerPhase { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .down + + case 2: return .move + + case 3: return .up + + case 4: return .cancel + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: PointerPhase, into buf: inout [UInt8]) { + switch value { + + + case .down: + writeInt(&buf, Int32(1)) + + + case .move: + writeInt(&buf, Int32(2)) + + + case .up: + writeInt(&buf, Int32(3)) + + + case .cancel: + writeInt(&buf, Int32(4)) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypePointerPhase_lift(_ buf: RustBuffer) throws -> PointerPhase { + return try FfiConverterTypePointerPhase.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypePointerPhase_lower(_ value: PointerPhase) -> RustBuffer { + return FfiConverterTypePointerPhase.lower(value) +} + + + + public enum PullRequestState: Equatable, Hashable { case `open` @@ -11776,6 +13069,7 @@ public enum RowKind: Equatable, Hashable { case chip case image case working + case visual @@ -11809,6 +13103,8 @@ public struct FfiConverterTypeRowKind: FfiConverterRustBuffer { case 6: return .working + case 7: return .visual + default: throw UniffiInternalError.unexpectedEnumCase } } @@ -11840,6 +13136,10 @@ public struct FfiConverterTypeRowKind: FfiConverterRustBuffer { case .working: writeInt(&buf, Int32(6)) + + case .visual: + writeInt(&buf, Int32(7)) + } } } @@ -12297,25 +13597,121 @@ public struct FfiConverterTypeStreamSpeed: FfiConverterRustBuffer { let variant: Int32 = try readInt(&buf) switch variant { - case 1: return .realistic + case 1: return .realistic + + case 2: return .fast + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: StreamSpeed, into buf: inout [UInt8]) { + switch value { + + + case .realistic: + writeInt(&buf, Int32(1)) + + + case .fast: + writeInt(&buf, Int32(2)) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeStreamSpeed_lift(_ buf: RustBuffer) throws -> StreamSpeed { + return try FfiConverterTypeStreamSpeed.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeStreamSpeed_lower(_ value: StreamSpeed) -> RustBuffer { + return FfiConverterTypeStreamSpeed.lower(value) +} + + + +/** + * How a surface takes touches. + */ + +public enum SurfaceGesture: Equatable, Hashable { + + /** + * Not interactive: drags scroll the transcript. + */ + case none + /** + * Taps only. + */ + case tap + /** + * Takes drags in any direction (orbits, pointer-driven drawings). + */ + case drag + /** + * Takes mostly-horizontal drags (scrubbing a chart); vertical ones + * scroll. + */ + case horizontalDrag + + + + + +} + +#if compiler(>=6) +extension SurfaceGesture: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeSurfaceGesture: FfiConverterRustBuffer { + typealias SwiftType = SurfaceGesture + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SurfaceGesture { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .none + + case 2: return .tap - case 2: return .fast + case 3: return .drag + + case 4: return .horizontalDrag default: throw UniffiInternalError.unexpectedEnumCase } } - public static func write(_ value: StreamSpeed, into buf: inout [UInt8]) { + public static func write(_ value: SurfaceGesture, into buf: inout [UInt8]) { switch value { - case .realistic: + case .none: writeInt(&buf, Int32(1)) - case .fast: + case .tap: writeInt(&buf, Int32(2)) + + case .drag: + writeInt(&buf, Int32(3)) + + + case .horizontalDrag: + writeInt(&buf, Int32(4)) + } } } @@ -12324,15 +13720,15 @@ public struct FfiConverterTypeStreamSpeed: FfiConverterRustBuffer { #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeStreamSpeed_lift(_ buf: RustBuffer) throws -> StreamSpeed { - return try FfiConverterTypeStreamSpeed.lift(buf) +public func FfiConverterTypeSurfaceGesture_lift(_ buf: RustBuffer) throws -> SurfaceGesture { + return try FfiConverterTypeSurfaceGesture.lift(buf) } #if swift(>=5.8) @_documentation(visibility: private) #endif -public func FfiConverterTypeStreamSpeed_lower(_ value: StreamSpeed) -> RustBuffer { - return FfiConverterTypeStreamSpeed.lower(value) +public func FfiConverterTypeSurfaceGesture_lower(_ value: SurfaceGesture) -> RustBuffer { + return FfiConverterTypeSurfaceGesture.lower(value) } @@ -12427,6 +13823,94 @@ public func FfiConverterTypeTranscriptScale_lower(_ value: TranscriptScale) -> R +public enum VisualShape: Equatable, Hashable { + + case rect(x: Float, y: Float, w: Float, h: Float, radius: Float, color: Rgba + ) + /** + * A polyline: `points` holds x, y pairs. + */ + case path(points: [Float], closed: Bool, fill: Rgba?, stroke: Rgba?, lineWidth: Float, + /** + * Dash and gap length. + */dash: Float? + ) + + + + + +} + +#if compiler(>=6) +extension VisualShape: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeVisualShape: FfiConverterRustBuffer { + typealias SwiftType = VisualShape + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> VisualShape { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .rect(x: try FfiConverterFloat.read(from: &buf), y: try FfiConverterFloat.read(from: &buf), w: try FfiConverterFloat.read(from: &buf), h: try FfiConverterFloat.read(from: &buf), radius: try FfiConverterFloat.read(from: &buf), color: try FfiConverterTypeRgba.read(from: &buf) + ) + + case 2: return .path(points: try FfiConverterSequenceFloat.read(from: &buf), closed: try FfiConverterBool.read(from: &buf), fill: try FfiConverterOptionTypeRgba.read(from: &buf), stroke: try FfiConverterOptionTypeRgba.read(from: &buf), lineWidth: try FfiConverterFloat.read(from: &buf), dash: try FfiConverterOptionFloat.read(from: &buf) + ) + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: VisualShape, into buf: inout [UInt8]) { + switch value { + + + case let .rect(x,y,w,h,radius,color): + writeInt(&buf, Int32(1)) + FfiConverterFloat.write(x, into: &buf) + FfiConverterFloat.write(y, into: &buf) + FfiConverterFloat.write(w, into: &buf) + FfiConverterFloat.write(h, into: &buf) + FfiConverterFloat.write(radius, into: &buf) + FfiConverterTypeRgba.write(color, into: &buf) + + + case let .path(points,closed,fill,stroke,lineWidth,dash): + writeInt(&buf, Int32(2)) + FfiConverterSequenceFloat.write(points, into: &buf) + FfiConverterBool.write(closed, into: &buf) + FfiConverterOptionTypeRgba.write(fill, into: &buf) + FfiConverterOptionTypeRgba.write(stroke, into: &buf) + FfiConverterFloat.write(lineWidth, into: &buf) + FfiConverterOptionFloat.write(dash, into: &buf) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVisualShape_lift(_ buf: RustBuffer) throws -> VisualShape { + return try FfiConverterTypeVisualShape.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeVisualShape_lower(_ value: VisualShape) -> RustBuffer { + return FfiConverterTypeVisualShape.lower(value) +} + + + + public enum VoiceCallPhase: Equatable, Hashable { case connecting @@ -13194,6 +14678,11 @@ public enum WidgetKind: Equatable, Hashable { * gradient). */ case shimmer + /** + * An agent visual; `TranscriptView::visual(id)` is its live session. + */ + case visual(id: String + ) @@ -13248,6 +14737,9 @@ public struct FfiConverterTypeWidgetKind: FfiConverterRustBuffer { case 12: return .shimmer + case 13: return .visual(id: try FfiConverterString.read(from: &buf) + ) + default: throw UniffiInternalError.unexpectedEnumCase } } @@ -13321,6 +14813,11 @@ public struct FfiConverterTypeWidgetKind: FfiConverterRustBuffer { case .shimmer: writeInt(&buf, Int32(12)) + + case let .visual(id): + writeInt(&buf, Int32(13)) + FfiConverterString.write(id, into: &buf) + } } } @@ -13413,6 +14910,30 @@ fileprivate struct FfiConverterOptionInt64: FfiConverterRustBuffer { } } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionFloat: FfiConverterRustBuffer { + typealias SwiftType = Float? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterFloat.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterFloat.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + #if swift(>=5.8) @_documentation(visibility: private) #endif @@ -13557,6 +15078,30 @@ fileprivate struct FfiConverterOptionTypeUploadProgress: FfiConverterRustBuffer } } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeVisualSession: FfiConverterRustBuffer { + typealias SwiftType = VisualSession? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeVisualSession.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeVisualSession.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + #if swift(>=5.8) @_documentation(visibility: private) #endif @@ -13629,6 +15174,30 @@ fileprivate struct FfiConverterOptionTypeContextUsage: FfiConverterRustBuffer { } } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeFrameImage: FfiConverterRustBuffer { + typealias SwiftType = FrameImage? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeFrameImage.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeFrameImage.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + #if swift(>=5.8) @_documentation(visibility: private) #endif @@ -13725,6 +15294,30 @@ fileprivate struct FfiConverterOptionTypePullRequest: FfiConverterRustBuffer { } } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeRgba: FfiConverterRustBuffer { + typealias SwiftType = Rgba? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeRgba.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeRgba.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + #if swift(>=5.8) @_documentation(visibility: private) #endif @@ -14765,6 +16358,81 @@ fileprivate struct FfiConverterSequenceTypeUserInputQuestion: FfiConverterRustBu } } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterSequenceTypeVisualControl: FfiConverterRustBuffer { + typealias SwiftType = [VisualControl] + + public static func write(_ value: [VisualControl], into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + for item in value { + FfiConverterTypeVisualControl.write(item, into: &buf) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [VisualControl] { + let len: Int32 = try readInt(&buf) + var seq = [VisualControl]() + seq.reserveCapacity(Int(len)) + for _ in 0 ..< len { + seq.append(try FfiConverterTypeVisualControl.read(from: &buf)) + } + return seq + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterSequenceTypeVisualRun: FfiConverterRustBuffer { + typealias SwiftType = [VisualRun] + + public static func write(_ value: [VisualRun], into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + for item in value { + FfiConverterTypeVisualRun.write(item, into: &buf) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [VisualRun] { + let len: Int32 = try readInt(&buf) + var seq = [VisualRun]() + seq.reserveCapacity(Int(len)) + for _ in 0 ..< len { + seq.append(try FfiConverterTypeVisualRun.read(from: &buf)) + } + return seq + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterSequenceTypeVisualSurface: FfiConverterRustBuffer { + typealias SwiftType = [VisualSurface] + + public static func write(_ value: [VisualSurface], into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + for item in value { + FfiConverterTypeVisualSurface.write(item, into: &buf) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [VisualSurface] { + let len: Int32 = try readInt(&buf) + var seq = [VisualSurface]() + seq.reserveCapacity(Int(len)) + for _ in 0 ..< len { + seq.append(try FfiConverterTypeVisualSurface.read(from: &buf)) + } + return seq + } +} + #if swift(>=5.8) @_documentation(visibility: private) #endif @@ -14790,6 +16458,31 @@ fileprivate struct FfiConverterSequenceTypeWidget: FfiConverterRustBuffer { } } +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterSequenceTypeVisualShape: FfiConverterRustBuffer { + typealias SwiftType = [VisualShape] + + public static func write(_ value: [VisualShape], into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + for item in value { + FfiConverterTypeVisualShape.write(item, into: &buf) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [VisualShape] { + let len: Int32 = try readInt(&buf) + var seq = [VisualShape]() + seq.reserveCapacity(Int(len)) + for _ in 0 ..< len { + seq.append(try FfiConverterTypeVisualShape.read(from: &buf)) + } + return seq + } +} + #if swift(>=5.8) @_documentation(visibility: private) #endif @@ -15601,12 +17294,18 @@ private let initializationResult: InitializationResult = { if (uniffi_zeron_mobile_checksum_method_transcriptview_set_viewport() != 43528) { return InitializationResult.apiChecksumMismatch } + if (uniffi_zeron_mobile_checksum_method_transcriptview_set_visual_source() != 22890) { + return InitializationResult.apiChecksumMismatch + } if (uniffi_zeron_mobile_checksum_method_transcriptview_toggle() != 51051) { return InitializationResult.apiChecksumMismatch } if (uniffi_zeron_mobile_checksum_method_transcriptview_toggle_detail() != 17098) { return InitializationResult.apiChecksumMismatch } + if (uniffi_zeron_mobile_checksum_method_transcriptview_visual() != 55147) { + return InitializationResult.apiChecksumMismatch + } if (uniffi_zeron_mobile_checksum_method_orbrenderer_next_frame() != 6064) { return InitializationResult.apiChecksumMismatch } @@ -15616,6 +17315,33 @@ private let initializationResult: InitializationResult = { if (uniffi_zeron_mobile_checksum_method_orbrenderer_set_orb() != 18211) { return InitializationResult.apiChecksumMismatch } + if (uniffi_zeron_mobile_checksum_method_visuallistener_display() != 33821) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visuallistener_frame() != 6525) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_activate() != 24011) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_attach() != 52504) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_detach() != 6554) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_pointer() != 37846) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_set_appearance() != 2236) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_slide() != 15641) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_zeron_mobile_checksum_method_visualsession_title() != 12743) { + return InitializationResult.apiChecksumMismatch + } if (uniffi_zeron_mobile_checksum_constructor_captionfader_new() != 15591) { return InitializationResult.apiChecksumMismatch } @@ -15636,6 +17362,7 @@ private let initializationResult: InitializationResult = { uniffiCallbackInitLayoutListener() uniffiCallbackInitPlatformMeasurer() uniffiCallbackInitUploadProgress() + uniffiCallbackInitVisualListener() uniffiCallbackInitVoiceMediaListener() uniffiCallbackInitVoiceSessionListener() return InitializationResult.ok diff --git a/apps/ios/Zeron/Session/CoreSessionSource.swift b/apps/ios/Zeron/Session/CoreSessionSource.swift index 3e96d3c5c9..adde56aa26 100644 --- a/apps/ios/Zeron/Session/CoreSessionSource.swift +++ b/apps/ios/Zeron/Session/CoreSessionSource.swift @@ -11,7 +11,10 @@ final class CoreSessionSource: SessionSource { private let chatId: String private var token: AnyObject? private var appToken: AnyObject? - private var hostDevice = "" + private var hostDevice = "" { + didSet { if hostDevice != oldValue { connectVisuals() } } + } + private weak var engine: TranscriptView? init(app: AppModel, client: CoreClient, handle: SessionHandle, chatId: String) { self.app = app @@ -25,9 +28,17 @@ final class CoreSessionSource: SessionSource { func attach(_ engine: TranscriptView) { _ = engine.attach(client: client, chatId: chatId) + self.engine = engine + connectVisuals() handle.setViewAttached(attached: true) } + /// Agent visuals load their scripts from the chat's host device. + private func connectVisuals() { + guard let engine, !hostDevice.isEmpty else { return } + engine.setVisualSource(client: client, deviceId: hostDevice, pixelScale: Float(max(2, UITraitCollection.current.displayScale))) + } + func detach() { handle.setViewAttached(attached: false) } diff --git a/apps/ios/Zeron/Transcript/RowView.swift b/apps/ios/Zeron/Transcript/RowView.swift index 8d4e7b4560..d41c43de62 100644 --- a/apps/ios/Zeron/Transcript/RowView.swift +++ b/apps/ios/Zeron/Transcript/RowView.swift @@ -36,6 +36,8 @@ protocol RowViewDelegate: AnyObject { func rowView(_ view: RowView, toggleDetail detail: UInt64, open: Bool) func rowView(_ view: RowView, open url: URL) func rowView(_ view: RowView, imageFor reference: String, into imageView: UIImageView) + /// The live view for an agent visual (kept across row reuse). + func rowView(_ view: RowView, visualFor id: String) -> UIView? } /// A reusable transcript row: canvas + horizontal scrollers + native widgets. @@ -53,7 +55,7 @@ final class RowView: UIView { textElement?.accessibilityFrameInContainerSpace = bounds // Controls and links first: accessibility hit-testing takes the // first match, and the row's text element spans the whole row. - return widgetViews.filter { $0 is UIControl || $0.gestureRecognizers?.isEmpty == false } + return widgetViews.filter { $0 is UIControl || $0 is VisualView || $0.gestureRecognizers?.isEmpty == false } + positionedLinkElements() + (textElement.map { [$0] } ?? []) } @@ -94,6 +96,17 @@ final class RowView: UIView { var uploadProgress: Double? { didSet { if uploadProgress != oldValue { uploadRings.forEach { $0.progress = uploadProgress } } } } + /// Pooled rows stay in the list, hidden: their visuals pause. + override var isHidden: Bool { + didSet { if isHidden != oldValue { updateVisuals() } } + } + + private func updateVisuals() { + for case let visual as VisualView in widgetViews where visual.superview === self { + visual.active = !isHidden + } + } + var key: UInt64 { model?.display.key ?? 0 } var version: UInt64 { model?.display.version ?? 0 } var kind: RowKind = .markdown @@ -150,6 +163,7 @@ final class RowView: UIView { case .chip: "row-chip" case .image: "row-image" case .working: "row-working" + case .visual: "row-visual" } element.accessibilityTraits = .staticText element.accessibilityFrameInContainerSpace = bounds @@ -192,7 +206,11 @@ final class RowView: UIView { private func layoutWidgets(_ d: RowDisplay) { uploadRings.removeAll() - for v in widgetViews { v.removeFromSuperview() } + // Only views still here: a visual's view moves between rows, and a + // row reused later mustn't pull it out of the one showing it now. + for v in widgetViews where v.superview === self || scrollers.contains(where: { $0 === v.superview }) { + v.removeFromSuperview() + } widgetViews.removeAll(keepingCapacity: true) for w in d.widgets { let rect = CGRect(x: CGFloat(w.x), y: CGFloat(w.y), width: CGFloat(w.w), height: CGFloat(w.h)) @@ -277,6 +295,9 @@ final class RowView: UIView { view = b case .shimmer: view = ShimmerView(model: model!, rect: rect) + case let .visual(id): + view = delegate?.rowView(self, visualFor: id) ?? UIView() + (view as? VisualView)?.active = !isHidden } view.frame = rect host.addSubview(view) diff --git a/apps/ios/Zeron/Transcript/TranscriptListView.swift b/apps/ios/Zeron/Transcript/TranscriptListView.swift index 514b7d8244..b4066153fd 100644 --- a/apps/ios/Zeron/Transcript/TranscriptListView.swift +++ b/apps/ios/Zeron/Transcript/TranscriptListView.swift @@ -12,6 +12,9 @@ final class TranscriptListView: UIScrollView, RowViewDelegate, UIScrollViewDeleg private var visible: [UInt64: RowView] = [:] private var pool: [RowView] = [] private var cache: [ModelKey: RowModel] = [:] + /// Agent visuals by id: one live view each, moved between reused rows so + /// its painted state survives row reconfigures. + private var visuals: [String: VisualView] = [:] private var memoryObserver: NSObjectProtocol? deinit { @@ -597,6 +600,18 @@ final class TranscriptListView: UIScrollView, RowViewDelegate, UIScrollViewDeleg imageLoader?(reference, imageView) } + func rowView(_ view: RowView, visualFor id: String) -> UIView? { + if let live = visuals[id] { return live } + guard let session = engine.visual(id: id) else { return nil } + let live = VisualView(session: session, scale: traitCollection.displayScale) + live.openURL = { [weak self, weak live] url in + guard let self, let row = live?.superview as? RowView else { return } + self.rowView(row, open: url) + } + visuals[id] = live + return live + } + fileprivate func row(at point: CGPoint) -> RowView? { visible.values.first { !$0.isHidden && $0.frame.contains(point) } } diff --git a/apps/ios/Zeron/Transcript/VisualView.swift b/apps/ios/Zeron/Transcript/VisualView.swift new file mode 100644 index 0000000000..89c47bd7d0 --- /dev/null +++ b/apps/ios/Zeron/Transcript/VisualView.swift @@ -0,0 +1,525 @@ +import CoreText +import UIKit + +/// An agent visual in the transcript. Its Rust worker (`VisualSession`) runs +/// the script, lays it out, draws its canvases and charts and renders its +/// scenes and shaders; this view only shows what arrives and forwards +/// touches. Paint is rasterized on the worker's thread, so the main thread +/// just swaps layer contents. +final class VisualView: UIView { + let session: VisualSession + var openURL: ((URL) -> Void)? + + private let sink: VisualSink + private let base = CALayer() + private var surfaces: [UInt64: VisualSurfaceView] = [:] + private var controls: [UInt64: VisualControlView] = [:] + private(set) var display: VisualDisplay? + private var attached = false + private var foreground = UIApplication.shared.applicationState != .background + private var observers: [NSObjectProtocol] = [] + + init(session: VisualSession, scale: CGFloat) { + self.session = session + sink = VisualSink(scale: scale) + super.init(frame: .zero) + sink.view = self + base.contentsGravity = .topLeft + base.contentsScale = scale + layer.addSublayer(base) + clipsToBounds = false + accessibilityLabel = session.title() + accessibilityIdentifier = "transcript-visual" + let center = NotificationCenter.default + observers = [ + center.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: .main) { [weak self] _ in + self?.foreground = false + self?.updateAttachment() + }, + center.addObserver(forName: UIApplication.willEnterForegroundNotification, object: nil, queue: .main) { [weak self] _ in + self?.foreground = true + self?.updateAttachment() + }, + ] + } + + required init?(coder: NSCoder) { fatalError() } + + deinit { + observers.forEach(NotificationCenter.default.removeObserver) + } + + /// Whether the row showing this view is on screen (pooled rows stay in + /// the window, hidden). + var active = true { + didSet { if active != oldValue { updateAttachment() } } + } + + override func didMoveToWindow() { + super.didMoveToWindow() + // Rows re-add their widgets on every configure: settle on the next + // turn of the run loop, so a move between rows doesn't detach. + DispatchQueue.main.async { [weak self] in self?.updateAttachment() } + } + + /// Frames flow only while the visual can be seen. + private func updateAttachment() { + let wanted = window != nil && active && foreground + guard wanted != attached else { return } + attached = wanted + if wanted { + session.attach(listener: sink, dark: traitCollection.userInterfaceStyle == .dark, reduceMotion: UIAccessibility.isReduceMotionEnabled) + } else { + session.detach() + } + } + + override func traitCollectionDidChange(_ previous: UITraitCollection?) { + super.traitCollectionDidChange(previous) + if previous?.userInterfaceStyle != traitCollection.userInterfaceStyle { + session.setAppearance(dark: traitCollection.userInterfaceStyle == .dark) + } + } + + /// A new layout: the static paint, surface frames and touch targets. + fileprivate func apply(_ d: VisualDisplay, image: CGImage?) { + CATransaction.begin() + CATransaction.setDisableActions(true) + defer { CATransaction.commit() } + display = d + base.frame = CGRect(x: 0, y: 0, width: CGFloat(d.width), height: CGFloat(d.height)) + base.contents = image + + var liveSurfaces = Set() + for s in d.surfaces { + liveSurfaces.insert(s.id) + let view = surfaces[s.id] ?? { + let v = VisualSurfaceView(id: s.id, session: session) + insertSubview(v, at: 0) + surfaces[s.id] = v + return v + }() + view.frame = CGRect(x: CGFloat(s.x), y: CGFloat(s.y), width: CGFloat(s.w), height: CGFloat(s.h)) + view.gesture = s.gesture + } + for (id, view) in surfaces where !liveSurfaces.contains(id) { + view.removeFromSuperview() + surfaces[id] = nil + } + + var liveControls = Set() + for c in d.controls { + liveControls.insert(c.id) + let view = controls[c.id] ?? { + let v = VisualControlView(id: c.id, session: session) + addSubview(v) + controls[c.id] = v + return v + }() + view.frame = CGRect(x: CGFloat(c.x), y: CGFloat(c.y), width: CGFloat(c.w), height: CGFloat(c.h)) + view.configure(c, openURL: { [weak self] url in self?.openURL?(url) }) + } + for (id, view) in controls where !liveControls.contains(id) { + view.removeFromSuperview() + controls[id] = nil + } + // Later controls above earlier ones, as laid out (a tappable stack + // under the controls inside it). + for c in d.controls { + if let view = controls[c.id] { bringSubviewToFront(view) } + } + } + + fileprivate func apply(frame id: UInt64, image: CGImage?, paint: CGImage?) { + guard let view = surfaces[id] else { return } + CATransaction.begin() + CATransaction.setDisableActions(true) + view.show(image: image, paint: paint) + CATransaction.commit() + } + + override var accessibilityElements: [Any]? { + get { controls.values.sorted { ($0.frame.minY, $0.frame.minX) < ($1.frame.minY, $1.frame.minX) } } + set {} + } +} + +/// Receives the worker's output on its thread, rasterizes it there, and +/// hands the main thread only the latest of each. +private final class VisualSink: VisualListener, @unchecked Sendable { + weak var view: VisualView? + private let scale: CGFloat + private let fonts = StyleFonts() + private let lock = NSLock() + private var sizes: [UInt64: CGSize] = [:] + private var pendingDisplay: (VisualDisplay, CGImage?)? + private var pendingFrames: [UInt64: (CGImage?, CGImage?)] = [:] + private var scheduled = false + + init(scale: CGFloat) { + self.scale = scale + } + + func display(display: VisualDisplay) { + fonts.update(display.styles) + let size = CGSize(width: CGFloat(display.width), height: CGFloat(display.height)) + let image = VisualPainter.render(display.paint, size: size, scale: scale, fonts: fonts) + lock.lock() + sizes = Dictionary(uniqueKeysWithValues: display.surfaces.map { ($0.id, CGSize(width: CGFloat($0.w), height: CGFloat($0.h))) }) + pendingDisplay = (display, image) + schedule() + lock.unlock() + } + + func frame(frame: SurfaceFrame) { + fonts.update(frame.styles) + lock.lock() + let size = sizes[frame.surface] + lock.unlock() + guard let size else { return } + let image = frame.image.flatMap(VisualPainter.image) + let paint = frame.paint.shapes.isEmpty && frame.paint.runs.isEmpty + ? nil + : VisualPainter.render(frame.paint, size: size, scale: scale, fonts: fonts) + lock.lock() + pendingFrames[frame.surface] = (image, paint) + schedule() + lock.unlock() + } + + /// Called with the lock held. + private func schedule() { + guard !scheduled else { return } + scheduled = true + DispatchQueue.main.async { [self] in + lock.lock() + let display = pendingDisplay + let frames = pendingFrames + pendingDisplay = nil + pendingFrames = [:] + scheduled = false + lock.unlock() + guard let view else { return } + if let (d, image) = display { view.apply(d, image: image) } + for (id, (image, paint)) in frames { view.apply(frame: id, image: image, paint: paint) } + } + } +} + +/// Draws Rust-placed shapes and runs. Pure CoreGraphics: safe off the main +/// thread. +enum VisualPainter { + static func color(_ c: Rgba) -> CGColor { + CGColor(srgbRed: CGFloat(c.r), green: CGFloat(c.g), blue: CGFloat(c.b), alpha: CGFloat(c.a)) + } + + private static let space = CGColorSpace(name: CGColorSpace.sRGB)! + private static let bitmap = CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue + + /// A worker frame (premultiplied BGRA) as an image, without copying. + static func image(_ frame: FrameImage) -> CGImage? { + guard frame.width > 0, frame.height > 0, + let provider = CGDataProvider(data: frame.bgra as CFData) + else { return nil } + return CGImage( + width: Int(frame.width), height: Int(frame.height), bitsPerComponent: 8, bitsPerPixel: 32, + bytesPerRow: Int(frame.width) * 4, space: space, bitmapInfo: CGBitmapInfo(rawValue: bitmap), + provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent + ) + } + + static func render(_ paint: VisualPaint, size: CGSize, scale: CGFloat, fonts: StyleFonts) -> CGImage? { + let width = Int((size.width * scale).rounded(.up)) + let height = Int((size.height * scale).rounded(.up)) + guard width > 0, height > 0, + let ctx = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: space, bitmapInfo: bitmap) + else { return nil } + // Top-left origin in points, like the display list. + ctx.translateBy(x: 0, y: CGFloat(height)) + ctx.scaleBy(x: scale, y: -scale) + draw(paint, in: ctx, fonts: fonts) + return ctx.makeImage() + } + + static func draw(_ paint: VisualPaint, in ctx: CGContext, fonts: StyleFonts) { + ctx.setLineJoin(.round) + ctx.setLineCap(.round) + for shape in paint.shapes { + switch shape { + case let .rect(x, y, w, h, radius, c): + let rect = CGRect(x: CGFloat(x), y: CGFloat(y), width: CGFloat(w), height: CGFloat(h)) + let r = min(CGFloat(radius), rect.width / 2, rect.height / 2) + ctx.setFillColor(color(c)) + if r > 0 { + ctx.addPath(CGPath(roundedRect: rect, cornerWidth: r, cornerHeight: r, transform: nil)) + ctx.fillPath() + } else { + ctx.fill(rect) + } + case let .path(points, closed, fill, stroke, lineWidth, dash): + guard points.count >= 4 else { continue } + let path = CGMutablePath() + path.move(to: CGPoint(x: CGFloat(points[0]), y: CGFloat(points[1]))) + var i = 2 + while i + 1 < points.count { + path.addLine(to: CGPoint(x: CGFloat(points[i]), y: CGFloat(points[i + 1]))) + i += 2 + } + if closed { path.closeSubpath() } + if let fill { + ctx.addPath(path) + ctx.setFillColor(color(fill)) + ctx.fillPath() + } + if let stroke { + ctx.addPath(path) + ctx.setStrokeColor(color(stroke)) + ctx.setLineWidth(CGFloat(lineWidth)) + if let dash { ctx.setLineDash(phase: 0, lengths: [CGFloat(dash), CGFloat(dash)]) } + ctx.strokePath() + if dash != nil { ctx.setLineDash(phase: 0, lengths: []) } + } + } + } + let text = paint.text as NSString + for run in paint.runs { + let range = NSRange(location: Int(run.start), length: Int(run.len)) + guard range.location + range.length <= text.length, let font = fonts.font(run.style) else { continue } + let s = NSAttributedString(string: text.substring(with: range), attributes: [ + .font: font.font, + .ligature: font.ligatures ? 1 : 0, + .foregroundColor: color(run.color), + ]) + let line = CTLineCreateWithAttributedString(s) + ctx.saveGState() + ctx.textMatrix = .identity + ctx.translateBy(x: CGFloat(run.x), y: CGFloat(run.baseline)) + ctx.scaleBy(x: 1, y: -1) + ctx.textPosition = .zero + CTLineDraw(line, ctx) + ctx.restoreGState() + } + } +} + +/// A canvas, chart, scene or shader: the worker's image under its vector +/// paint, taking touches the way the surface asks. +private final class VisualSurfaceView: UIView, UIGestureRecognizerDelegate { + let id: UInt64 + private weak var session: VisualSession? + private let imageLayer = CALayer() + private let paintLayer = CALayer() + private var recognizers: [UIGestureRecognizer] = [] + + var gesture: SurfaceGesture = .none { + didSet { if gesture != oldValue || recognizers.isEmpty { installGestures() } } + } + + init(id: UInt64, session: VisualSession) { + self.id = id + self.session = session + super.init(frame: .zero) + for l in [imageLayer, paintLayer] { + l.contentsGravity = .resize + layer.addSublayer(l) + } + installGestures() + } + + required init?(coder: NSCoder) { fatalError() } + + override func layoutSubviews() { + super.layoutSubviews() + CATransaction.begin() + CATransaction.setDisableActions(true) + imageLayer.frame = bounds + paintLayer.frame = bounds + CATransaction.commit() + } + + func show(image: CGImage?, paint: CGImage?) { + imageLayer.contents = image + paintLayer.contents = paint + } + + private func installGestures() { + recognizers.forEach(removeGestureRecognizer) + recognizers = [] + isUserInteractionEnabled = gesture != .none + switch gesture { + case .none: + break + case .tap: + let tap = UITapGestureRecognizer(target: self, action: #selector(tapped(_:))) + recognizers = [tap] + case .drag: + // Tracks from touch-down, so drags here never scroll the transcript. + let press = UILongPressGestureRecognizer(target: self, action: #selector(tracked(_:))) + press.minimumPressDuration = 0 + press.allowableMovement = .greatestFiniteMagnitude + recognizers = [press] + case .horizontalDrag: + let pan = UIPanGestureRecognizer(target: self, action: #selector(tracked(_:))) + pan.delegate = self + let tap = UITapGestureRecognizer(target: self, action: #selector(tapped(_:))) + recognizers = [pan, tap] + } + recognizers.forEach(addGestureRecognizer) + } + + /// Mostly-horizontal drags scrub; vertical ones scroll the transcript. + /// (UIKit asks this view about ancestors' recognizers too: only judge + /// our own.) + override func gestureRecognizerShouldBegin(_ g: UIGestureRecognizer) -> Bool { + guard let pan = g as? UIPanGestureRecognizer, pan.view === self, gesture == .horizontalDrag else { + return super.gestureRecognizerShouldBegin(g) + } + let v = pan.velocity(in: self) + return abs(v.x) > abs(v.y) + } + + private func send(_ phase: PointerPhase, _ p: CGPoint) { + session?.pointer(surface: id, phase: phase, x: Float(p.x), y: Float(p.y)) + } + + private var readoutTimer: Timer? + + @objc private func tapped(_ g: UITapGestureRecognizer) { + let p = g.location(in: self) + send(.down, p) + guard gesture == .horizontalDrag else { return send(.up, p) } + // A tap on a chart shows its readout for a moment. + readoutTimer?.invalidate() + readoutTimer = Timer.scheduledTimer(withTimeInterval: 2.5, repeats: false) { [weak self] _ in + self?.send(.up, p) + } + } + + @objc private func tracked(_ g: UIGestureRecognizer) { + let p = g.location(in: self) + switch g.state { + case .began: send(.down, p) + case .changed: send(.move, p) + case .ended: send(.up, p) + case .cancelled, .failed: send(.cancel, p) + default: break + } + } +} + +/// A touch target over control art the worker painted: buttons, tabs, +/// toggles, sliders, links and tappable stacks. +private final class VisualControlView: UIControl { + let id: UInt64 + private weak var session: VisualSession? + private var kind: ControlKind = .tap + private var openURL: ((URL) -> Void)? + private let wash = CALayer() + + init(id: UInt64, session: VisualSession) { + self.id = id + self.session = session + super.init(frame: .zero) + wash.backgroundColor = Palette.controlFill.cgColor + wash.opacity = 0 + wash.cornerCurve = .continuous + layer.addSublayer(wash) + isAccessibilityElement = true + addAction(UIAction { [weak self] _ in self?.activated() }, for: .touchUpInside) + } + + required init?(coder: NSCoder) { fatalError() } + + func configure(_ c: VisualControl, openURL: @escaping (URL) -> Void) { + kind = c.kind + self.openURL = openURL + accessibilityLabel = c.label + accessibilityValue = nil + switch c.kind { + case .button, .tap: + accessibilityTraits = .button + case let .tab(selected): + accessibilityTraits = selected ? [.button, .selected] : .button + case let .toggle(on): + accessibilityTraits = .button + accessibilityValue = on ? "On" : "Off" + case let .slider(_, _, _, _, _, _, _, shown): + accessibilityTraits = .adjustable + accessibilityValue = shown + case .link: + accessibilityTraits = .link + } + let radius: CGFloat = switch c.kind { + case .tap: 6 + case .button, .tab: 8 + default: 0 + } + wash.cornerRadius = radius + setNeedsLayout() + } + + override func layoutSubviews() { + super.layoutSubviews() + wash.frame = bounds + } + + override func traitCollectionDidChange(_ previous: UITraitCollection?) { + super.traitCollectionDidChange(previous) + wash.backgroundColor = Palette.controlFill.cgColor + } + + override var isHighlighted: Bool { + didSet { + switch kind { + case .slider, .toggle: return + default: wash.opacity = isHighlighted ? 1 : 0 + } + } + } + + /// A finger-sized target around small art. + override func point(inside point: CGPoint, with event: UIEvent?) -> Bool { + let dy = max(0, (44 - bounds.height) / 2) + return bounds.insetBy(dx: -4, dy: -dy).contains(point) + } + + private func activated() { + switch kind { + case .button, .tap: + session?.activate(control: id) + case .tab, .toggle: + UISelectionFeedbackGenerator().selectionChanged() + session?.activate(control: id) + case let .link(url): + if let url = URL(string: url) { openURL?(url) } + case .slider: + break + } + } + + // Sliders track the finger along their track (in the visual's points). + override func beginTracking(_ touch: UITouch, with event: UIEvent?) -> Bool { + if case .slider = kind { slide(to: touch) } + return true + } + + override func continueTracking(_ touch: UITouch, with event: UIEvent?) -> Bool { + if case .slider = kind { slide(to: touch) } + return true + } + + private func slide(to touch: UITouch) { + guard case let .slider(_, _, _, _, _, trackX, trackW, _) = kind, let parent = superview, trackW > 0 else { return } + let x = Float(touch.location(in: parent).x) + session?.slide(control: id, fraction: max(0, min(1, (x - trackX) / trackW))) + } + + override func accessibilityIncrement() { nudge(1) } + override func accessibilityDecrement() { nudge(-1) } + + private func nudge(_ direction: Float) { + guard case let .slider(_, min, max, step, fraction, _, _, _) = kind, max > min else { return } + let by = step > 0 ? Float(step / (max - min)) : 0.1 + session?.slide(control: id, fraction: Swift.max(0, Swift.min(1, fraction + direction * by))) + } +} diff --git a/apps/ios/ZeronUITests/VisualFlowTests.swift b/apps/ios/ZeronUITests/VisualFlowTests.swift new file mode 100644 index 0000000000..10290c4284 --- /dev/null +++ b/apps/ios/ZeronUITests/VisualFlowTests.swift @@ -0,0 +1,51 @@ +import XCTest + +/// Agent visuals in the demo transcript: rendered by their Rust workers, +/// driven through the touch targets laid over their controls. +final class VisualFlowTests: XCTestCase { + override func setUp() { + continueAfterFailure = false + } + + private func snapshot(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + func testDashboardControlsDriveTheScript() { + let app = XCUIApplication() + app.launchArguments = ["-demo", "-route", "chat:chat-visuals"] + app.launch() + + let toggle = app.buttons["Show table"] + XCTAssertTrue(toggle.waitForExistence(timeout: 15)) + XCTAssertEqual(toggle.value as? String, "Off") + snapshot(app, "dashboard") + + toggle.tap() + // The script re-renders, the worker lays out again, the target updates. + let on = NSPredicate(format: "value == 'On'") + expectation(for: on, evaluatedWith: toggle) + waitForExpectations(timeout: 5) + snapshot(app, "dashboard-table") + + let slider = app.otherElements["Smoothing window"].firstMatch.exists + ? app.otherElements["Smoothing window"].firstMatch + : app.descendants(matching: .any)["Smoothing window"].firstMatch + XCTAssertTrue(slider.exists) + XCTAssertEqual(slider.value as? String, "3") + // A touch on the track moves the thumb there (1…7 in steps of 1). + slider.coordinate(withNormalizedOffset: CGVector(dx: 0.84, dy: 0.5)).tap() + expectation(for: NSPredicate(format: "value == '6'"), evaluatedWith: slider) + waitForExpectations(timeout: 5) + + let tab = app.buttons["30d"] + XCTAssertTrue(tab.exists) + tab.tap() + expectation(for: NSPredicate(format: "selected == true"), evaluatedWith: tab) + waitForExpectations(timeout: 5) + snapshot(app, "dashboard-30d") + } +} diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index 2187afa34a..3727a58ff0 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -32,6 +32,7 @@ async-trait.workspace = true [dev-dependencies] tempfile.workspace = true +zeron-visual.workspace = true zeron-sync = { workspace = true, features = ["mock-server"] } tokio-tungstenite.workspace = true async-trait.workspace = true diff --git a/crates/client/src/demo/fixtures.rs b/crates/client/src/demo/fixtures.rs index 2a7b9fb750..8030551aa1 100644 --- a/crates/client/src/demo/fixtures.rs +++ b/crates/client/src/demo/fixtures.rs @@ -174,6 +174,16 @@ pub(crate) fn chats() -> Vec { "Found 412 items older than 90 days.", ) }, + DemoChat { + last_ago_ms: 2 * MIN, + ..chat( + "chat-visuals", + None, + MAC, + "Agent visuals", + "A dashboard, a canvas, a scene and a shader.", + ) + }, DemoChat { last_ago_ms: 10 * MIN, ..chat( diff --git a/crates/client/src/demo/mod.rs b/crates/client/src/demo/mod.rs index 528fb66ad2..021367ee9b 100644 --- a/crates/client/src/demo/mod.rs +++ b/crates/client/src/demo/mod.rs @@ -12,6 +12,7 @@ mod fixtures; mod png; mod transcripts; +mod visuals; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -1066,6 +1067,9 @@ impl DemoHost { if let Some(bytes) = lock(&self.uploads).get(path) { return Ok(bytes.clone()); } + if let Some(bytes) = visuals::read(path) { + return Ok(Arc::new(bytes)); + } transcripts::DEMO_IMAGES .iter() .find(|(p, ..)| *p == path) diff --git a/crates/client/src/demo/transcripts.rs b/crates/client/src/demo/transcripts.rs index e99691811d..64151ef4ce 100644 --- a/crates/client/src/demo/transcripts.rs +++ b/crates/client/src/demo/transcripts.rs @@ -563,6 +563,49 @@ find ~/Downloads -maxdepth 1 -mtime +90 ! -name '_archive-*' \ ] } +fn visuals(host: &str, now: i64) -> Vec { + let local = super::visuals::local(); + if !local.is_empty() { + let mut parts = Vec::new(); + for (i, visual) in local.iter().enumerate() { + parts.push(text(&format!("t{i}"), &format!("**{}**", visual.title))); + parts.push(MessagePart::Visual { + id: visual.id.clone(), + path: visual.path.clone(), + title: visual.title.clone(), + }); + } + return vec![ + user("m1", now - 300_000, "Show the desktop's visuals."), + assistant("m2", host, now - 290_000, parts), + ]; + } + vec![ + user("m1", now - 300_000, "Show me what visuals can do."), + assistant( + "m2", + host, + now - 290_000, + vec![ + text("t0", "A canvas that follows your finger:"), + super::visuals::part(1), + text( + "t1", + "A scene you can orbit, with metal, glass and a light:", + ), + super::visuals::part(2), + text("t2", "A shader; drag to move the light."), + super::visuals::part(3), + text( + "t3", + "And a dashboard: switch the range, scrub the chart, and toggle the table.", + ), + super::visuals::part(0), + ], + ), + ] +} + fn cjk(host: &str, now: i64) -> Vec { vec![ user( @@ -619,6 +662,7 @@ pub(crate) fn fixture(chat_id: &str, host: &str, last_activity: i64) -> Vec scroll(host, now), "chat-home" => home(host, now), "chat-cjk" => cjk(host, now), + "chat-visuals" => visuals(host, now), "chat-deploy" => short( host, now, diff --git a/crates/client/src/demo/visuals.rs b/crates/client/src/demo/visuals.rs new file mode 100644 index 0000000000..5e64628fc4 --- /dev/null +++ b/crates/client/src/demo/visuals.rs @@ -0,0 +1,163 @@ +//! Agent visuals the demo serves (`show_visual` scripts), one of each kind: +//! a dashboard with controls, a pointer-driven canvas, an orbiting scene and +//! a light shader. +//! +//! To preview visuals a desktop rendered instead, point +//! `ZERON_DEMO_VISUALS` at a JSON list of `{ "id", "title", "path" }` (the +//! part's id and title, and its script on disk): the demo chat shows those, +//! reading each script and the files stored beside it. + +use std::sync::OnceLock; + +use zeron_doc::parts::MessagePart; + +#[derive(serde::Deserialize)] +pub(crate) struct Local { + pub id: String, + pub title: String, + pub path: String, +} + +/// Visuals listed by `ZERON_DEMO_VISUALS` (empty without it). +pub(crate) fn local() -> &'static [Local] { + static LOCAL: OnceLock> = OnceLock::new(); + LOCAL.get_or_init(|| { + let Ok(list) = std::env::var("ZERON_DEMO_VISUALS") else { + return Vec::new(); + }; + std::fs::read(&list) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() + }) +} + +/// A listed visual's script, manifest or one of its assets. +fn local_file(path: &str) -> Option> { + local().iter().find_map(|visual| { + let stem = visual.path.strip_suffix("-visual.luau"); + let beside = path == visual.path + || stem.is_some_and(|stem| { + path == format!("{stem}-assets.json") + || path + .strip_prefix(&format!("{stem}-asset-")) + .is_some_and(|name| !name.contains('/')) + }); + beside.then(|| std::fs::read(path).ok()).flatten() + }) +} + +pub(crate) const DEMO_VISUALS: &[(&str, &str, &str)] = &[ + ("v_demo_dash", "Latency dashboard", DASHBOARD), + ("v_demo_canvas", "Orbiting particles", CANVAS), + ("v_demo_scene", "Materials", SCENE), + ("v_demo_shader", "Light", SHADER), +]; + +pub(crate) fn path(id: &str) -> String { + format!("/Users/dev/.zeron/uploads/{id}-visual.luau") +} + +/// The file at `path`, if it's a demo visual's. +pub(crate) fn read(path: &str) -> Option> { + DEMO_VISUALS + .iter() + .find(|(id, ..)| self::path(id) == path) + .map(|(_, _, script)| script.as_bytes().to_vec()) + .or_else(|| local_file(path)) +} + +pub(crate) fn part(index: usize) -> MessagePart { + let (id, title, _) = DEMO_VISUALS[index]; + MessagePart::Visual { + id: id.to_owned(), + path: path(id), + title: title.to_owned(), + } +} + +const DASHBOARD: &str = r#" +local s = ui.state { range = "7d", table = false, window = 3 } +local days = { "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" } +local p95 = { ["7d"] = { 182, 175, 190, 168, 160, 171, 158 }, ["30d"] = { 210, 198, 190, 181, 176, 170, 162 } } +local p50 = { ["7d"] = { 92, 88, 95, 84, 80, 86, 79 }, ["30d"] = { 104, 99, 95, 91, 88, 85, 81 } } +return function() + local rows = {} + for i, day in ipairs(days) do + rows[i] = { day, p95[s.range][i], p50[s.range][i] } + end + return ui.column { gap = 14, + ui.row { gap = 16, + ui.stat { value = "158 ms", label = "p95 latency", delta = "−8%", tone = "success" }, + ui.stat { value = "99.95%", label = "uptime" }, + ui.stat { value = "4.2k", label = "req / min" }, + }, + ui.tabs { options = { "7d", "30d" }, value = s.range, on_change = function(v) s.range = v end }, + ui.chart { type = "area", height = 160, x = days, unit = "ms", + series = { { name = "p95", values = p95[s.range] }, { name = "p50", values = p50[s.range] } } }, + ui.slider { label = "Smoothing window", value = s.window, min = 1, max = 7, step = 1, + on_change = function(v) s.window = v end }, + ui.row { gap = 12, align = "center", + ui.toggle { label = "Show table", value = s.table, on_change = function(v) s.table = v end }, + ui.button { label = "Reset", on_click = function() s.window = 3 end }, + }, + s.table and ui.table { columns = { "Day", { label = "p95", align = "end" }, { label = "p50", align = "end" } }, rows = rows } + or ui.space(0), + } +end +"#; + +const CANVAS: &str = r#" +return ui.column { gap = 6, + ui.text { "Drag across it", size = "sm", color = "muted" }, + ui.canvas { height = 220, animate = true, draw = function(g, w, h, t) + local p = g.pointer + local cx = p and p.x or w / 2 + math.cos(t * 0.7) * w / 4 + local cy = p and p.y or h / 2 + math.sin(t * 0.9) * h / 4 + for i = 1, 36 do + local a = t * 0.8 + i * (math.pi * 2 / 36) + local r = 34 + 22 * math.sin(t * 2 + i * 0.7) + g:circle(cx + math.cos(a) * r, cy + math.sin(a) * r, 3.5, { fill = "chart" .. (i % 6 + 1) }) + end + g:circle(cx, cy, 6, { fill = "accent" }) + end }, +} +"#; + +const SCENE: &str = r#" +return ui.scene { height = 260, orbit = true, spin = 0.25, draw = function(s) + s:box { at = { 0, -0.05, 0 }, size = { 5, 0.1, 3.5 }, color = "muted" } + s:box { at = { -1.4, 0.5, 0 }, size = 1, color = "chart1", roughness = 0.35 } + s:sphere { at = { 0, 0.6, 0.4 }, radius = 0.6, color = "chart2", metal = 1, roughness = 0.25 } + s:box { at = { 1.4, 0.6, 0 }, size = { 0.9, 1.2, 0.9 }, color = "chart3", glass = true, dispersion = 1 } + s:sphere { at = { 1.4, 1.6, -1 }, radius = 0.2, color = "chart4", glow = 4 } +end } +"#; + +const SHADER: &str = r#" +return ui.shader { height = 220, bloom = 1.2, code = [[ +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y; + vec2 m = iMouse.x > 0.0 ? (iMouse.xy - 0.5 * iResolution.xy) / iResolution.y : vec2(0.0); + vec3 col = vec3(0.0); + for (int i = 0; i < 6; i++) { + float fi = float(i); + vec2 c = m + 0.28 * vec2(cos(iTime * 0.9 + fi), sin(iTime * 1.2 + fi * 2.0)); + float d = length(uv - c); + col += 0.010 / (d * d + 0.002) * (0.5 + 0.5 * cos(6.2831 * (fi / 6.0 + vec3(0.0, 0.33, 0.67)))); + } + fragColor = vec4(col, 1.0); +} +]] } +"#; + +#[cfg(test)] +mod tests { + #[test] + fn demo_visuals_pass_the_engine_check() { + for (id, _, script) in super::DEMO_VISUALS { + zeron_visual::check(script, zeron_visual::seed_for(id)) + .unwrap_or_else(|e| panic!("{id}: {e}")); + } + } +} diff --git a/crates/client/tests/demo.rs b/crates/client/tests/demo.rs index 100765a737..301ab3d4b2 100644 --- a/crates/client/tests/demo.rs +++ b/crates/client/tests/demo.rs @@ -61,7 +61,13 @@ fn front_page_mirrors_the_desktop_sidebar() { // Recency order; archived + child chats never appear. assert_eq!( ids(&ws.front.recent), - ["chat-cjk", "chat-home", "chat-deploy", "chat-blog"] + [ + "chat-visuals", + "chat-cjk", + "chat-home", + "chat-deploy", + "chat-blog" + ] ); assert_eq!(ids(&ws.archived), ["chat-oklch", "chat-presence"]); assert_eq!(ids(ws.children("chat-veil")), ["chat-side"]); @@ -105,7 +111,7 @@ fn front_page_mirrors_the_desktop_sidebar() { let cjk = ws.session("chat-cjk").unwrap(); assert_eq!(cjk.project.as_ref().unwrap().color_index, zeron.color_index); assert_ne!(ws.project("space-edge").unwrap().group_key, zeron.group_key); - assert_eq!(ids(&ws.projectless), ["chat-home"]); + assert_eq!(ids(&ws.projectless), ["chat-visuals", "chat-home"]); assert!(ws.devices.iter().any(|d| d.is_self && d.id == "ios-test")); assert!(!ws.device("dev-studio").unwrap().online); diff --git a/crates/engine/src/rpc.rs b/crates/engine/src/rpc.rs index 93bb4d0047..79fb23ed35 100644 --- a/crates/engine/src/rpc.rs +++ b/crates/engine/src/rpc.rs @@ -502,6 +502,10 @@ struct ReadAttachmentChunkParams { } const _: () = assert!(zeron_proto::VISUAL_MAX_BYTES == zeron_visual::MAX_SCRIPT_BYTES); +const _: () = assert!(zeron_proto::VISUAL_MAX_ASSETS == zeron_visual::assets::MAX_ASSETS); +const _: () = assert!(zeron_proto::VISUAL_MAX_ASSET_BYTES == zeron_visual::assets::MAX_ASSET_BYTES); +const _: () = + assert!(zeron_proto::VISUAL_MAX_TOTAL_ASSET_BYTES == zeron_visual::assets::MAX_TOTAL_ASSET_BYTES); #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -509,6 +513,57 @@ struct PublishVisualParams { chat_id: String, title: String, script: String, + /// Files the script uses, base64 by name. + #[serde(default)] + assets: std::collections::BTreeMap, +} + +fn no_turn_for_visual() -> RpcError { + RpcError::Failed( + "This chat has no running turn on this device; show_visual only works from inside a turn.".into(), + ) +} + +/// Decode and bound a visual's assets. +fn decode_visual_assets( + assets: &std::collections::BTreeMap, +) -> Result)>, RpcError> { + use base64::Engine as _; + use zeron_visual::assets::{MAX_ASSET_BYTES, MAX_ASSETS, MAX_TOTAL_ASSET_BYTES, valid_name}; + if assets.len() > MAX_ASSETS { + return Err(RpcError::BadParams(format!( + "a visual takes at most {MAX_ASSETS} assets" + ))); + } + let mut total = 0; + let mut out = Vec::with_capacity(assets.len()); + let mut seen = std::collections::HashSet::new(); + for (name, data) in assets { + if !valid_name(name) { + return Err(RpcError::BadParams(format!( + "asset name '{name}' must be a plain file name" + ))); + } + // Stored as files, and file systems often ignore case. + if !seen.insert(name.to_ascii_lowercase()) { + return Err(RpcError::BadParams(format!( + "asset names must differ by more than case ('{name}')" + ))); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(data) + .map_err(|e| RpcError::BadParams(format!("asset '{name}' is not base64: {e}")))?; + total += bytes.len(); + if bytes.len() > MAX_ASSET_BYTES || total > MAX_TOTAL_ASSET_BYTES { + return Err(RpcError::BadParams(format!( + "asset '{name}' is too large ({} MB each, {} MB together)", + MAX_ASSET_BYTES / (1024 * 1024), + MAX_TOTAL_ASSET_BYTES / (1024 * 1024) + ))); + } + out.push((name.clone(), bytes)); + } + Ok(out) } #[derive(Debug, Deserialize)] @@ -3718,19 +3773,39 @@ impl RpcService for EngineRpc { zeron_proto::VISUAL_MAX_BYTES ))); } + // Before decoding and storing anything (checked again below). + if !self.sessions.has_run(&p.chat_id) { + return Err(no_turn_for_visual()); + } let id = format!("v:{}", crate::new_id()); + let assets = decode_visual_assets(&p.assets)?; // Run the script the way the desktop will (setup, render, // every draw, every control) before anyone sees it, so the // agent gets its errors with line numbers and can fix them. let uploads = self.uploads.clone(); let (stored_id, script) = (id.clone(), p.script); let (path, report) = tokio::task::spawn_blocking(move || { - let checked = zeron_visual::check(&script, zeron_visual::seed_for(&stored_id)); + let files = zeron_visual::Assets::new( + assets + .iter() + .map(|(name, bytes)| (name.clone(), bytes.as_slice().into())) + .collect(), + ); + let checked = zeron_visual::check_with_assets( + &script, + zeron_visual::seed_for(&stored_id), + files, + ); let report = checked .map_err(|e| RpcError::BadParams(format!("visual script failed: {e}")))?; - let path = uploads - .store_visual(&stored_id, &script) + // Assets first: a script is only ever found with its files. + uploads + .store_visual_assets(&stored_id, &assets) .map_err(|e| RpcError::Failed(e.to_string()))?; + let path = uploads.store_visual(&stored_id, &script).map_err(|e| { + uploads.remove_visual(&stored_id); + RpcError::Failed(e.to_string()) + })?; Ok::<_, RpcError>((path, report)) }) .await @@ -3741,12 +3816,8 @@ impl RpcService for EngineRpc { title: title.clone(), }; if !self.sessions.inject_event(&p.chat_id, event) { - let _ = std::fs::remove_file(&path); - return Err(RpcError::Failed( - "This chat has no running turn on this device; show_visual only works \ - from inside a turn." - .into(), - )); + self.uploads.remove_visual(&id); + return Err(no_turn_for_visual()); } RpcReply::value(&serde_json::json!({ "id": id, diff --git a/crates/engine/src/sessions.rs b/crates/engine/src/sessions.rs index 6dfc5254fa..8c1b91edd5 100644 --- a/crates/engine/src/sessions.rs +++ b/crates/engine/src/sessions.rs @@ -1123,6 +1123,11 @@ impl SessionsEngine { /// Fold an engine-made event (a `show_visual` page) into the chat's live /// run, after whatever the harness streamed so far. Returns `false` when /// the chat has no run on this device. + /// Whether the chat has a run on this device (one `inject_event` can reach). + pub fn has_run(&self, chat_id: &str) -> bool { + lock(&self.inner.runs).contains_key(chat_id) + } + pub fn inject_event(&self, chat_id: &str, event: AgentEvent) -> bool { lock(&self.inner.runs) .get(chat_id) diff --git a/crates/engine/src/uploads.rs b/crates/engine/src/uploads.rs index fdc4eaf178..a8ec7974d1 100644 --- a/crates/engine/src/uploads.rs +++ b/crates/engine/src/uploads.rs @@ -528,6 +528,25 @@ fn mime_by_ext(path: &Path) -> Option<&'static str> { mod tests { use super::*; + #[test] + fn visual_assets_sit_beside_the_script_and_leave_with_it() { + let dir = tempfile::tempdir().unwrap(); + let uploads = Uploads::from_root(dir.path()); + let assets = vec![ + ("prism.glb".to_owned(), b"glb".to_vec()), + ("sky.hdr".to_owned(), b"hdr".to_vec()), + ]; + uploads.store_visual_assets("v:abc", &assets).unwrap(); + let script = uploads.store_visual("v:abc", "return ui.text 'hi'").unwrap(); + let stem = script.strip_suffix("-visual.luau").unwrap(); + let names: Vec = + serde_json::from_slice(&std::fs::read(format!("{stem}-assets.json")).unwrap()).unwrap(); + assert_eq!(names, ["prism.glb", "sky.hdr"]); + assert_eq!(std::fs::read(format!("{stem}-asset-sky.hdr")).unwrap(), b"hdr"); + uploads.remove_visual("v:abc"); + assert_eq!(std::fs::read_dir(uploads.dir()).unwrap().count(), 0); + } + #[test] fn sanitize_names() { assert_eq!(sanitize("../../etc/passwd"), "passwd"); @@ -893,6 +912,62 @@ impl Uploads { Ok(path.to_string_lossy().to_string()) } + /// Store the files a visual's script uses, beside it: each as + /// `{id}-asset-{name}`, and `{id}-assets.json` listing their names + /// (clients find it from the script's path). + pub fn store_visual_assets(&self, id: &str, assets: &[(String, Vec)]) -> Result<(), EngineError> { + use std::io::Write; + if assets.is_empty() { + return Ok(()); + } + std::fs::create_dir_all(self.dir())?; + let id = sanitize(id); + let write = |name: &str, bytes: &[u8]| -> Result<(), EngineError> { + let mut temporary = tempfile::NamedTempFile::new_in(self.dir())?; + temporary.write_all(bytes)?; + temporary + .persist(self.dir().join(name)) + .map_err(|err| EngineError::Other(err.to_string()))?; + Ok(()) + }; + // All or nothing: a failure part way removes what was written. + let mut written = Vec::with_capacity(assets.len() + 1); + let stored = (|| { + let mut names = Vec::with_capacity(assets.len()); + for (name, bytes) in assets { + let name = sanitize(name); + let file = format!("{id}-asset-{name}"); + write(&file, bytes)?; + written.push(file); + names.push(name); + } + let manifest = serde_json::to_vec(&names).map_err(|err| EngineError::Other(err.to_string()))?; + write(&format!("{id}-assets.json"), &manifest) + })(); + if stored.is_err() { + for file in written { + let _ = std::fs::remove_file(self.dir().join(file)); + } + } + stored + } + + /// Remove a visual's script and its assets. + pub fn remove_visual(&self, id: &str) { + let id = sanitize(id); + let _ = std::fs::remove_file(self.dir().join(format!("{id}-visual.luau"))); + let manifest = self.dir().join(format!("{id}-assets.json")); + if let Some(names) = std::fs::read(&manifest) + .ok() + .and_then(|bytes| serde_json::from_slice::>(&bytes).ok()) + { + for name in names { + let _ = std::fs::remove_file(self.dir().join(format!("{id}-asset-{}", sanitize(&name)))); + } + } + let _ = std::fs::remove_file(manifest); + } + /// Copy a Codex-owned raster into the active profile without exposing the /// original path to the journal, document, RPC readers, or relay. pub fn import_generated_image( diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml index ca060c2b72..323bdf6da5 100644 --- a/crates/mcp/Cargo.toml +++ b/crates/mcp/Cargo.toml @@ -13,6 +13,7 @@ tokio.workspace = true serde.workspace = true serde_json.workspace = true anyhow.workspace = true +base64 = "0.22" tracing.workspace = true uuid.workspace = true chrono.workspace = true diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs index c8cb3905d9..6d4b87d801 100644 --- a/crates/mcp/src/tools.rs +++ b/crates/mcp/src/tools.rs @@ -235,7 +235,12 @@ fn catalog() -> Vec { "type": "string", "description": "The Luau visual script: setup at the top level, ending with `return function() return ui.column { ... } end`." }, - "title": { "type": "string", "description": "Short name for the visual." } + "title": { "type": "string", "description": "Short name for the visual." }, + "assets": { + "type": "object", + "additionalProperties": { "type": "string" }, + "description": "Files the script uses, by the name it uses them: { \"prism.glb\": \"models/prism.glb\", \"sky.hdr\": \"/tmp/sky.hdr\" }. Paths are on this machine, relative to your working directory. glTF models (.glb, or .gltf with everything embedded), PNG, JPEG, WebP, and Radiance .hdr images; at most 16 files, 8 MB each, 10 MB together." + } }, "required": ["script", "title"] }), @@ -319,14 +324,27 @@ f = function(x, z) return y end or values = { { ... } }, detail = 32, colors = { (a ramp by height), wire = bool }, s:mesh { vertices = { { x, y, z }, ... }, faces = { { 1, 2, 3 }, ... }, \ color }, s:line(a, b, { color, width }), s:path(points, { color, width }), s:point(at, { radius, \ color, hint = \"shown on hover\" }), s:label(at, text, { color, size }), s:grid { size = 10, step = 1 }, \ -s:axes { size = 1, labels = { \"x\", \"y\", \"z\" } }.\n\ +s:axes { size = 1, labels = { \"x\", \"y\", \"z\" } }. Physical colors may be \"#rrggbb\" in scenes \ +(a material's own color: gold, glass, wood), and only there.\n\ +- Assets: pass real files in show_visual's assets (name -> path) and scripts use them by name. \ +s:model { asset = \"prism.glb\", at, rotate = { x, y, z } degrees, scale, fit = 2 (center it and size \ +its longest side), color and material props override the model's own } draws a glTF model with its \ +materials and textures. Solids take texture = \"wood.jpg\" (multiplied by color; mesh takes uvs = { { u, v \ +}, ... } per vertex). s:environment { asset = \"studio.hdr\", intensity = 1, background = false } lights \ +the scene from an equirectangular image (an HDR is best) that metal and glass reflect and glass bends; \ +background = true also draws it behind the scene. s:sprite { at, size = 48 (pixels), color, glow = 1, \ +asset = \"flare.png\" } adds a camera-facing light (a flare or glint; a soft round glow without an \ +asset) that blooms. ui.shader takes textures = { \"rock.jpg\", ... } as iTexture0-iTexture3 \ +(repeating, mipmapped). Find or make assets with your own tools (download, convert KTX2 or EXR to PNG \ +or HDR) before calling.\n\ - ui.shader { height = 360, code = [[ GLSL ]], buffers = { [[ GLSL ]], ... }, inputs = { speed = \ s.speed }, blend = \"alpha|light|opaque\", bloom = 0-3, orbit = bool, spin, camera = { yaw, pitch, \ zoom }, animate = true }: GPU shaders for what geometry can't express: raymarched SDFs and \ fractals, light and optics (beams, refraction, dispersion), volumes, fluids, particles, generative \ art. Write each pass as on Shadertoy: void mainImage(out vec4 fragColor, in vec2 fragCoord), with \ -iResolution (vec3, pixels), iTime, iTimeDelta, iFrame, iMouse (pixels from the bottom-left; xy while \ -pressed, zw where pressed), iCamera (vec4 yaw, pitch, zoom, 0; orbit = true lets the reader drag \ +iResolution (vec3: width and height in pixels, then the display's pixel density), iTime, \ +iTimeDelta, iFrame, iMouse (pixels from the bottom-left; xy where the pointer is, zw where it went \ +down while it's held, negative otherwise), iCamera (vec4 yaw, pitch, zoom, 0; orbit = true lets the reader drag \ it), every inputs name as a float (bind them to state so sliders and toggles drive the shader), \ iNoise (a tiling 256x256 random RGBA texture), and iChannel0-iChannel3 with iChannelResolution: \ buffers run first, in order, each into a float texture that every pass reads as iChannel (an \ @@ -336,13 +354,16 @@ background, vignette, or floor color. By default (blend = \"light\") the output brightness is opacity and black is clear, right for beams, glows, sparks, and light through glass. \ For solid shaded objects (fractals, SDF bodies) set blend = \"alpha\" and write alpha 0 where the ray \ misses. bloom = 0.6-1.5 adds a soft glow around bright light. \ +The width varies with the reader (a phone is about 350px wide, often narrower than the shader is \ +tall): scale the scene by min(iResolution.x, iResolution.y), not one axis, and keep the subject \ +centered with room on both sides, so it fits and stays centered at any aspect. \ GLSL errors come back with the pass and line number.\n\ - Helpers: ui.scale(d0, d1, r0, r1) returns a mapping function; ui.ticks(lo, hi, n); ui.fmt(n, { unit, \ prefix, decimals }) groups digits; ui.map(list, function(v, i) end); ui.range(a, b, step); ui.clamp; \ ui.lerp; ui.series(i) is \"chart\"; ui.color(token, alpha); ui.mix(a, b, t). print(...) output comes \ back to you.\n\ \n\ -Colors are theme tokens, never hex or color names: text; muted (labels, axes, captions); faint \ +Colors are theme tokens, never hex or color names (except scene materials): text; muted (labels, axes, captions); faint \ (tertiary notes); accent (the one thing to notice); on-accent (text on an accent mark); success, \ warning, danger (status only); border (hairlines, gridlines); chart1, chart2, chart3, chart4, \ chart5, chart6 (series, in order; \ @@ -358,12 +379,82 @@ notes, lg and xl for headings, 2xl for a headline figure. The width is the reply as narrow as 360px), so lay out fluidly with grow, wrap, and grid, and give charts, canvases, and \ scenes heights of 160-420px."; +/// Read `show_visual`'s asset files (paths relative to the agent's working +/// directory), base64-encoded by name for the engine. +async fn read_visual_assets( + assets: &std::collections::BTreeMap, +) -> anyhow::Result> { + use base64::Engine as _; + use zeron_proto::{ + VISUAL_MAX_ASSET_BYTES as MAX_ASSET_BYTES, VISUAL_MAX_ASSETS as MAX_ASSETS, + VISUAL_MAX_TOTAL_ASSET_BYTES as MAX_TOTAL_ASSET_BYTES, + valid_visual_asset_name as valid_name, + }; + anyhow::ensure!( + assets.len() <= MAX_ASSETS, + "a visual takes at most {MAX_ASSETS} assets" + ); + let mut out = serde_json::Map::new(); + let mut total = 0; + let mut seen = std::collections::HashSet::new(); + for (name, path) in assets { + anyhow::ensure!( + valid_name(name), + "asset name '{name}' must be a plain file name (letters, digits, '.', '-', '_')" + ); + // Stored as files, and file systems often ignore case. + anyhow::ensure!( + seen.insert(name.to_ascii_lowercase()), + "asset names must differ by more than case ('{name}')" + ); + let metadata = tokio::fs::metadata(path) + .await + .with_context(|| format!("asset '{name}': can't read {path}"))?; + anyhow::ensure!(metadata.is_file(), "asset '{name}': {path} is not a file"); + anyhow::ensure!( + metadata.len() <= MAX_ASSET_BYTES as u64, + "asset '{name}' is over {} MB", + MAX_ASSET_BYTES / (1024 * 1024) + ); + // Bounded however the file changes (or lies about its size). + let mut bytes = Vec::new(); + { + use tokio::io::AsyncReadExt as _; + tokio::fs::File::open(path) + .await + .with_context(|| format!("asset '{name}': can't read {path}"))? + .take(MAX_ASSET_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .await + .with_context(|| format!("asset '{name}': can't read {path}"))?; + } + anyhow::ensure!( + bytes.len() <= MAX_ASSET_BYTES, + "asset '{name}' is over {} MB", + MAX_ASSET_BYTES / (1024 * 1024) + ); + total += bytes.len(); + anyhow::ensure!( + total <= MAX_TOTAL_ASSET_BYTES, + "a visual's assets total at most {} MB", + MAX_TOTAL_ASSET_BYTES / (1024 * 1024) + ); + out.insert( + name.clone(), + Value::String(base64::engine::general_purpose::STANDARD.encode(bytes)), + ); + } + Ok(out) +} + // ---- argument shapes --------------------------------------------------------- #[derive(Deserialize)] struct ShowVisualArgs { script: String, title: String, + #[serde(default)] + assets: std::collections::BTreeMap, } #[derive(Deserialize)] @@ -1122,6 +1213,7 @@ impl Tools { .chat_id .clone() .context("show_visual only works from an agent Zeron launched in a chat")?; + let assets = read_visual_assets(&args.assets).await?; let shown = self .zeron .call( @@ -1130,6 +1222,7 @@ impl Tools { "chatId": chat_id, "title": args.title, "script": args.script, + "assets": assets, }), ) .await?; diff --git a/crates/mobile/Cargo.toml b/crates/mobile/Cargo.toml index 1e1845b303..5587bd81fc 100644 --- a/crates/mobile/Cargo.toml +++ b/crates/mobile/Cargo.toml @@ -30,6 +30,8 @@ async-trait.workspace = true uniffi = "0.32" zeron-markdown.workspace = true zeron-text.workspace = true +zeron-visual.workspace = true +zeron-visual-gpu.workspace = true zeron-doc.workspace = true zeron-proto.workspace = true zeron-client.workspace = true diff --git a/crates/mobile/src/layout/display.rs b/crates/mobile/src/layout/display.rs index 89665d0506..1ac28ec122 100644 --- a/crates/mobile/src/layout/display.rs +++ b/crates/mobile/src/layout/display.rs @@ -193,6 +193,8 @@ pub enum WidgetKind { /// underneath; the painter re-draws those runs brighter through a moving /// gradient). Shimmer, + /// An agent visual; `TranscriptView::visual(id)` is its live session. + Visual { id: String }, } #[derive(Debug, Clone, PartialEq, uniffi::Record)] diff --git a/crates/mobile/src/layout/mod.rs b/crates/mobile/src/layout/mod.rs index b3c5dd8f28..8b88bff959 100644 --- a/crates/mobile/src/layout/mod.rs +++ b/crates/mobile/src/layout/mod.rs @@ -12,7 +12,7 @@ pub mod display; mod markdown; mod file_icons; mod rows; -mod style; +pub(crate) mod style; mod tools; use std::collections::HashMap; @@ -73,6 +73,19 @@ impl zeron_text::FallbackMeasurer for MeasurerBridge { } } +/// Faces and fallback measurement for one layout thread (a transcript, a +/// visual). +pub(crate) fn typography(text: &TextSystem) -> Typography { + let styles = Arc::new(Mutex::new(HashMap::new())); + let fallback = text.measurer.clone().map(|platform| { + Arc::new(MeasurerBridge { + platform, + styles: styles.clone(), + }) as Arc + }); + Typography::new(&text.faces, fallback, styles) +} + #[derive(uniffi::Record)] pub struct FaceData { pub role: FaceRole, @@ -82,7 +95,9 @@ pub struct FaceData { /// Registered faces + fallback measurer, shared by every transcript. #[derive(uniffi::Object)] pub struct TextSystem { - faces: Vec<(FaceRole, Arc>)>, + /// Leaked once here: every transcript and visual measures with the same + /// bytes (a font book holds its faces for the process). + faces: Vec<(FaceRole, &'static [u8])>, measurer: Option>, } @@ -91,7 +106,10 @@ impl TextSystem { #[uniffi::constructor] pub fn new(faces: Vec, measurer: Option>) -> Arc { Arc::new(Self { - faces: faces.into_iter().map(|f| (f.role, Arc::new(f.bytes))).collect(), + faces: faces + .into_iter() + .map(|f| (f.role, &*Box::leak(f.bytes.into_boxed_slice()))) + .collect(), measurer, }) } @@ -120,11 +138,15 @@ struct FrameRow { gap: Gap, y: f32, height: f32, + /// A visual's measured-height epoch when the frame was built. + epoch: u64, + /// The visual's height this frame was measured with (painted the same). + visual: Option, } impl FrameRow { fn version(&self) -> u64 { - self.core.version ^ ((self.gap as u64 + 1) << 58) + self.core.version ^ ((self.gap as u64 + 1) << 58) ^ self.epoch.wrapping_mul(0x9e37_79b9_7f4a_7c15) } } @@ -205,7 +227,7 @@ impl LayoutFrame { pub fn display(&self, index: u32) -> Option { let r = self.rows.get(index as usize)?; let mut out = DisplayBuilder::default(); - let height = place_row(&r.core, r.gap, Px(self.scale), self.width, Some(&mut out)); + let height = place_row(&r.core, r.gap, Px(self.scale), self.width, r.visual, Some(&mut out)); debug_assert!((height - r.height).abs() < 0.01, "paint/measure divergence"); Some(RowDisplay { key: r.core.key, @@ -331,6 +353,8 @@ enum Msg { Viewport { width: f32, scale: f32 }, Toggle(u64), ToggleDetail { row: u64, detail: u64, open: bool }, + /// A visual row's height changed (its key), or visuals got a source. + Visual(Option), Shutdown, } @@ -345,6 +369,7 @@ pub struct TranscriptView { shared: Arc, /// Live session subscription (Rust→Rust; rows never cross FFI). watch: Mutex>, + visuals: Arc, } #[uniffi::export] @@ -355,18 +380,42 @@ impl TranscriptView { let shared = Arc::new(Shared { frame: Mutex::new(Arc::new(LayoutFrame::empty())), }); + let remeasure = Mutex::new(tx.clone()); + let visuals = Arc::new(crate::visual::VisualHub::new( + text.clone(), + Arc::new(move |key| { + let _ = remeasure.lock().unwrap().send(Msg::Visual(Some(key))); + }), + )); let worker_shared = shared.clone(); + let worker_visuals = visuals.clone(); thread::Builder::new() .name("zeron-layout".into()) - .spawn(move || Worker::new(&text, worker_shared, listener).run(rx)) + .spawn(move || Worker::new(&text, worker_shared, listener, Some(worker_visuals)).run(rx)) .expect("spawn layout thread"); Arc::new(Self { tx: Mutex::new(tx), shared, watch: Mutex::new(None), + visuals, }) } + /// Where this transcript's visuals load their scripts from (the chat's + /// host device), and the screen's pixel density for their frames. + pub fn set_visual_source(&self, client: Arc, device_id: String, pixel_scale: f32) { + if device_id.is_empty() { + return; + } + self.visuals.set_source(client.client().clone(), device_id, pixel_scale); + self.send(Msg::Visual(None)); + } + + /// The live visual behind a `WidgetKind::Visual` (by its visual id). + pub fn visual(&self, id: String) -> Option> { + self.visuals.session(&id) + } + /// Follow a session's transcript: every snapshot (coalesced by the /// client) becomes a layout input. Returns false for an unknown chat. pub fn attach(&self, client: Arc, chat_id: String) -> bool { @@ -424,6 +473,7 @@ impl TranscriptView { pub fn close(&self) { self.watch.lock().unwrap().take(); + self.visuals.shutdown(); self.send(Msg::Shutdown); } } @@ -441,6 +491,7 @@ impl TranscriptView { impl Drop for TranscriptView { fn drop(&mut self) { + self.visuals.shutdown(); self.send(Msg::Shutdown); } } @@ -489,19 +540,18 @@ pub(crate) struct Worker { revision: u64, shared: Arc, listener: Option>, + visuals: Option>, } impl Worker { - fn new(text: &TextSystem, shared: Arc, listener: Arc) -> Self { - let styles = Arc::new(Mutex::new(HashMap::new())); - let fallback = text.measurer.clone().map(|platform| { - Arc::new(MeasurerBridge { - platform, - styles: styles.clone(), - }) as Arc - }); + fn new( + text: &TextSystem, + shared: Arc, + listener: Arc, + visuals: Option>, + ) -> Self { Self { - typo: Typography::new(&text.faces, fallback, styles), + typo: typography(text), cache: WidthCache::new(), builder: RowBuilder::default(), heights: HashMap::new(), @@ -510,6 +560,7 @@ impl Worker { revision: 0, shared, listener: Some(listener), + visuals, } } @@ -533,6 +584,7 @@ impl Worker { self.builder.expanded = old.expanded; self.builder.collapsed = old.collapsed; self.builder.detail_open = old.detail_open; + self.builder.visual_slots = old.visual_slots; self.heights.clear(); self.cache = WidthCache::new(); } @@ -544,6 +596,13 @@ impl Worker { self.builder.invalidate(row); dirty = true; } + Msg::Visual(key) => { + // Its row measures again from the visual's height. + if let Some(key) = key { + self.heights.remove(&key); + } + dirty = true; + } Msg::Toggle(key) => { if !self.builder.expanded.remove(&key) && !self.builder.collapsed.remove(&key) { // First toggle flips the row's default. @@ -597,9 +656,22 @@ impl Worker { let mut y = 0.0f32; let mut live = HashMap::with_capacity(placed.len()); for p in placed { + let (epoch, visual) = match &p.core.content { + rows::Content::Visual(slot) => { + if let Some(visuals) = &self.visuals { + visuals.ensure(slot, p.core.key, rows::column_width(px, self.width), self.typo.scale); + } + slot.snapshot() + } + _ => (0, None), + }; let height = match self.heights.remove(&p.core.key) { - Some(m) if Arc::ptr_eq(&m.core, &p.core) && m.gap == p.gap && m.width == self.width => m.height, - _ => place_row(&p.core, p.gap, px, self.width, None), + Some(m) + if Arc::ptr_eq(&m.core, &p.core) && m.gap == p.gap && m.width == self.width && visual.is_none() => + { + m.height + } + _ => place_row(&p.core, p.gap, px, self.width, visual, None), }; live.insert( p.core.key, @@ -615,6 +687,8 @@ impl Worker { gap: p.gap, y, height, + epoch, + visual, }); y += height; } @@ -641,4 +715,4 @@ impl Worker { } #[cfg(test)] -mod tests; +pub(crate) mod tests; diff --git a/crates/mobile/src/layout/rows.rs b/crates/mobile/src/layout/rows.rs index 272721dc85..a2ea5aae27 100644 --- a/crates/mobile/src/layout/rows.rs +++ b/crates/mobile/src/layout/rows.rs @@ -32,6 +32,7 @@ pub enum RowKind { Chip, Image, Working, + Visual, } /// A user message awaiting its host echo (client-minted id = the echo's id). @@ -92,6 +93,8 @@ pub(crate) enum Content { Chip(Chip), Image { reference: String }, Working { since_ms: Option, streaming: bool }, + /// An agent visual: its height comes from its worker. + Visual(Arc), } /// A width-independent row: identity, top gap class and prepared content. @@ -141,6 +144,7 @@ pub(crate) mod geom { pub const CHIP_LINE: f32 = 32.0; pub const IMAGE: f32 = 260.0; pub const WORKING: f32 = 36.0; + pub const VISUAL_LOADING: f32 = 18.0; } impl Gap { @@ -184,6 +188,9 @@ pub(crate) struct RowBuilder { pub collapsed: HashSet, /// Per-tool inline detail overrides (row detail key → open). pub detail_open: HashMap, + /// Visual rows' links to their workers, by visual id (kept across + /// rebuilds: the worker holds the same slot). + pub visual_slots: HashMap>, } pub(crate) fn quick_hash(s: &str) -> u64 { @@ -404,6 +411,23 @@ impl RowBuilder { }); rows.push(Placed { core, gap }); } + MessagePart::Visual { id, path, title } => { + let gap = gap_for(&rows, false); + let slot = self + .visual_slots + .entry(id.clone()) + .or_insert_with(|| crate::visual::VisualSlot::new(id, path, title)) + .clone(); + let core = Arc::new(RowCore { + key: row_key(&format!("{}#{}", entry.id, id)), + version: next_version(), + kind: RowKind::Visual, + entry_id: Arc::from(entry.id.as_str()), + content: Content::Visual(slot), + copy_text: title.clone(), + }); + rows.push(Placed { core, gap }); + } MessagePart::Input { id, questions, resolved, .. } => { let text = if *resolved { "Answered a question".to_owned() @@ -698,12 +722,27 @@ fn chip_row(ctx: &mut Ctx, entry_id: &str, part_id: &str, icon: &'static str, co } } +/// The message column's width in a viewport `width` wide. +pub(crate) fn column_width(px: Px, width: f32) -> f32 { + use geom::*; + // Comfortable measure on iPad/landscape: cap the column, center it. + (width - px.v(MARGIN_X) * 2.0).clamp(40.0, px.v(READING_WIDTH)) +} + /// Lays out (and optionally paints) one row at `width` (full viewport width). -pub(crate) fn place_row(core: &RowCore, gap: Gap, px: Px, width: f32, mut out: Option<&mut DisplayBuilder>) -> f32 { +/// `visual` is a visual row's measured height (its slot's, read with the +/// frame's version). +pub(crate) fn place_row( + core: &RowCore, + gap: Gap, + px: Px, + width: f32, + visual: Option, + mut out: Option<&mut DisplayBuilder>, +) -> f32 { use geom::*; let margin = px.v(MARGIN_X); - // Comfortable measure on iPad/landscape: cap the column, center it. - let cw = (width - margin * 2.0).clamp(40.0, px.v(READING_WIDTH)); + let cw = column_width(px, width); let x = ((width - cw) / 2.0).max(margin).floor(); let top = gap.px(px); let body = match &core.content { @@ -731,6 +770,14 @@ pub(crate) fn place_row(core: &RowCore, gap: Gap, px: Px, width: f32, mut out: O } side } + Content::Visual(slot) => { + // Until its worker has measured it: the loading line. + let h = visual.unwrap_or(px.v(VISUAL_LOADING)); + if let Some(out) = out { + out.widget(WidgetKind::Visual { id: slot.id.clone() }, (x, top, cw, h), None); + } + h + } Content::Working { since_ms, streaming } => { let h = px.v(WORKING); if let Some(out) = out { @@ -871,6 +918,6 @@ pub(crate) fn content_heap_bytes(content: &Content) -> usize { Content::Tools(t) => super::tools::heap_bytes(t), Content::Chip(c) => c.text.p.heap_bytes(), Content::Image { reference } => reference.len(), - Content::Working { .. } => 0, + Content::Working { .. } | Content::Visual(_) => 0, } } diff --git a/crates/mobile/src/layout/style.rs b/crates/mobile/src/layout/style.rs index 0dc8b602ea..18e8393136 100644 --- a/crates/mobile/src/layout/style.rs +++ b/crates/mobile/src/layout/style.rs @@ -149,14 +149,14 @@ pub(crate) struct Typography { impl Typography { pub fn new( - faces: &[(FaceRole, Arc>)], + faces: &[(FaceRole, &'static [u8])], fallback: Option>, registry: Arc>>, ) -> Self { let mut book = FontBook::new(); let mut ids = HashMap::new(); for (role, bytes) in faces { - if let Ok(id) = book.add_face(bytes.as_ref().clone()) { + if let Ok(id) = book.add_static_face(bytes) { ids.insert(*role, id); } } diff --git a/crates/mobile/src/layout/tests.rs b/crates/mobile/src/layout/tests.rs index d503e96903..9cd625eb0b 100644 --- a/crates/mobile/src/layout/tests.rs +++ b/crates/mobile/src/layout/tests.rs @@ -41,7 +41,7 @@ pub(crate) fn text_system() -> Arc { fn worker(width: f32) -> Worker { let ts = text_system(); - let mut w = Worker::new(&ts, Arc::new(Shared { frame: Mutex::new(Arc::new(LayoutFrame::empty())) }), Arc::new(Quiet)); + let mut w = Worker::new(&ts, Arc::new(Shared { frame: Mutex::new(Arc::new(LayoutFrame::empty())) }), Arc::new(Quiet), None); w.width = width; w } @@ -606,3 +606,45 @@ fn links_get_hit_regions() { assert!(!links.is_empty(), "no link hits for {md:?}"); } } + +#[test] +fn visual_rows_take_the_height_their_worker_measures() { + struct Ping(Mutex>); + impl LayoutListener for Ping { + fn frame_ready(&self, revision: u64) { + let _ = self.0.lock().unwrap().send(revision); + } + } + let (tx, frames) = mpsc::channel(); + let view = TranscriptView::new(text_system(), Arc::new(Ping(Mutex::new(tx)))); + let script = r#"return ui.column { gap = 10, ui.text "Hello", ui.canvas { height = 200, draw = function(g) end } }"#; + view.visuals.set_fetch(Arc::new(move |path: &str| (path == "up/v1-visual.luau").then(|| script.as_bytes().to_vec())), 2.0); + view.set_viewport(390.0, 1.0); + view.set_input(TranscriptInput { + entries: vec![Arc::new(SessionMessageEntry { + id: "a1".into(), + role: MessageRole::Assistant, + parts: vec![MessagePart::Visual { id: "v1".into(), path: "up/v1-visual.luau".into(), title: "Chart".into() }], + created_at: 0, + device_id: String::new(), + status: Some(MessageStatus::Complete), + continuation_of: None, + duration_ms: None, + })], + ..Default::default() + }); + // The loading line first, then the laid-out visual (text, gap, canvas). + let deadline = Instant::now() + std::time::Duration::from_secs(10); + let (frame, row) = loop { + let _ = frames.recv_timeout(deadline.saturating_duration_since(Instant::now())).expect("a frame"); + let frame = view.frame(); + if let Some(row) = frame.placement(0).filter(|p| p.kind == RowKind::Visual && p.height > 200.0) { + break (frame, row); + } + }; + let display = frame.display(row.index).unwrap(); + let widget = display.widgets.iter().find(|w| matches!(&w.kind, display::WidgetKind::Visual { id } if id == "v1")).expect("a visual widget"); + assert!((widget.y + widget.h - row.height).abs() < 0.5, "the widget fills the row below its gap"); + assert!(view.visual("v1".into()).is_some()); + view.close(); +} diff --git a/crates/mobile/src/lib.rs b/crates/mobile/src/lib.rs index f54d888654..275065fbdb 100644 --- a/crates/mobile/src/lib.rs +++ b/crates/mobile/src/lib.rs @@ -13,6 +13,7 @@ pub mod layout; pub mod caption; pub mod orb; pub mod wallpaper; +pub mod visual; /// Version handshake: the Swift/Kotlin bindings must match the linked library. #[uniffi::export] diff --git a/crates/mobile/src/visual/chart.rs b/crates/mobile/src/visual/chart.rs new file mode 100644 index 0000000000..231149860a --- /dev/null +++ b/crates/mobile/src/visual/chart.rs @@ -0,0 +1,658 @@ +//! Charts as vector paint in the chart's own coordinates: the desktop's +//! domains, ticks, label thinning and marks (`crates/ui/src/visuals.rs`). +//! The legend band above the plot doubles as the readout while a finger +//! scrubs the plot (the desktop's hover). + +use zeron_visual::{Align, Chart, ChartKind, TextSize, Weight}; + +use super::Rgba; +use super::paint::{Face, Paint, TextCtx}; +use super::palette::Palette; + +/// Gaps between the plot and its axis labels. +const AXIS_GAP: f32 = 8.0; +/// Between the legend band and the plot. +pub(crate) const LEGEND_GAP: f32 = 8.0; +const LEGEND_GAP_X: f32 = 14.0; +const LEGEND_GAP_Y: f32 = 2.0; +const SWATCH: f32 = 8.0; + +/// What a finger over the plot picks out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Hover { + pub category: Option, + pub point: Option<(usize, usize)>, +} + +fn domain(lo: f64, hi: f64, include_zero: bool) -> (f64, f64, Vec) { + let (mut lo, mut hi) = (lo, hi); + if include_zero { + lo = lo.min(0.0); + hi = hi.max(0.0); + } + if !lo.is_finite() || !hi.is_finite() { + return (0.0, 1.0, vec![0.0, 1.0]); + } + if (hi - lo).abs() < f64::EPSILON { + hi = lo + lo.abs().max(1.0); + } + let ticks = zeron_visual::nice_ticks(lo, hi, 4); + let step = if ticks.len() > 1 { + ticks[1] - ticks[0] + } else { + (hi - lo).max(1.0) + }; + let lo = (lo / step).floor() * step; + let hi = (hi / step).ceil() * step; + (lo, hi, zeron_visual::nice_ticks(lo, hi, 4)) +} + +fn value_range(chart: &Chart) -> (f64, f64) { + let (mut lo, mut hi) = (f64::INFINITY, f64::NEG_INFINITY); + if chart.kind == ChartKind::Scatter { + for (_, y) in chart.series.iter().flat_map(|s| &s.points) { + lo = lo.min(*y); + hi = hi.max(*y); + } + } else if chart.stacked { + let n = chart + .series + .iter() + .map(|s| s.values.len()) + .max() + .unwrap_or(0); + for i in 0..n { + let (mut pos, mut neg) = (0.0, 0.0); + for s in &chart.series { + match s.values.get(i).copied().flatten() { + Some(v) if v >= 0.0 => pos += v, + Some(v) => neg += v, + None => {} + } + } + lo = lo.min(neg); + hi = hi.max(pos); + } + } else { + for v in chart.series.iter().flat_map(|s| s.values.iter().flatten()) { + lo = lo.min(*v); + hi = hi.max(*v); + } + } + (chart.y_min.unwrap_or(lo), chart.y_max.unwrap_or(hi)) +} + +fn categories(chart: &Chart) -> usize { + chart + .series + .iter() + .map(|s| s.values.len()) + .max() + .unwrap_or(0) + .max(chart.x.len()) +} + +fn x_extent(chart: &Chart) -> (f64, f64) { + chart + .series + .iter() + .flat_map(|s| &s.points) + .fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), (x, _)| { + (lo.min(*x), hi.max(*x)) + }) +} + +fn xs() -> Face { + Face::new(TextSize::Xs, Weight::Regular) +} + +fn sm(weight: Weight) -> Face { + Face::new(TextSize::Sm, weight) +} + +fn measure(ctx: &mut TextCtx, text: &str, face: Face) -> f32 { + ctx.line(text, face, Rgba::default()).natural() +} + +fn show_legend(chart: &Chart) -> bool { + chart.legend && chart.series.len() > 1 && chart.series.iter().any(|s| !s.name.is_empty()) +} + +/// Legend entries flowed into rows at `width`: (x, row) per entry. +fn flow(widths: &[f32], width: f32) -> Vec<(f32, usize)> { + let (mut x, mut row) = (0.0f32, 0usize); + widths + .iter() + .map(|w| { + if x > 0.0 && x + w > width + 0.5 { + x = 0.0; + row += 1; + } + let at = (x, row); + x += w + LEGEND_GAP_X; + at + }) + .collect() +} + +/// The legend band's height at `width` (one line when there's no legend, +/// kept for the readout). +pub(crate) fn legend_height( + ctx: &mut TextCtx, + _palette: &Palette, + chart: &Chart, + width: f32, +) -> f32 { + let line = ctx.line_height(TextSize::Sm); + if !show_legend(chart) { + return line; + } + let widths: Vec = chart + .series + .iter() + .map(|s| SWATCH + 6.0 + measure(ctx, &s.name, sm(Weight::Regular))) + .collect(); + let rows = flow(&widths, width).last().map_or(0, |(_, row)| *row) + 1; + rows as f32 * line + (rows - 1) as f32 * LEGEND_GAP_Y +} + +struct Plan { + /// x, y, w, h in chart coordinates. + plot: [f32; 4], + lo: f64, + hi: f64, + ticks: Vec, + x_lo: f64, + x_hi: f64, + x_ticks: Vec, + label_stride: usize, +} + +fn plan(ctx: &mut TextCtx, chart: &Chart, width: f32, legend: f32) -> Plan { + let tick_line = ctx.line_height(TextSize::Xs); + let scatter = chart.kind == ChartKind::Scatter; + let horizontal = chart.kind == ChartKind::Bar && chart.horizontal; + let (lo, hi) = value_range(chart); + let (lo, hi, ticks) = domain(lo, hi, !scatter); + let (x_lo, x_hi, x_ticks) = if scatter { + let (a, b) = x_extent(chart); + domain(a, b, false) + } else { + (0.0, 1.0, Vec::new()) + }; + let n = categories(chart).max(1); + // The left gutter fits the widest value tick, or the category labels on + // a horizontal bar chart. + let left = if horizontal { + chart + .x + .iter() + .map(|l| measure(ctx, l, xs())) + .fold(0.0, f32::max) + .min(width * 0.35) + } else { + ticks + .iter() + .map(|t| measure(ctx, &chart.format.format(*t), xs())) + .fold(0.0, f32::max) + } + AXIS_GAP; + let top = legend + LEGEND_GAP + tick_line / 2.0; + let bottom = tick_line + AXIS_GAP / 2.0; + let right = if horizontal || scatter { 12.0 } else { 4.0 }; + let plot = [ + left, + top, + (width - left - right).max(0.0), + (legend + LEGEND_GAP + chart.height - bottom - top).max(0.0), + ]; + // Thin the category labels until they don't collide. + let label_stride = if horizontal || scatter { + 1 + } else { + let widest = chart + .x + .iter() + .map(|l| measure(ctx, l, xs())) + .fold(0.0, f32::max) + + 12.0; + let band = plot[2] / n as f32; + ((widest / band.max(1.0)).ceil() as usize).max(1) + }; + Plan { + plot, + lo, + hi, + ticks, + x_lo, + x_hi, + x_ticks, + label_stride, + } +} + +/// What's under (x, y) in chart coordinates. +pub(crate) fn hover_at( + ctx: &mut TextCtx, + chart: &Chart, + width: f32, + legend: f32, + [x, y]: [f32; 2], +) -> Hover { + let plan = plan(ctx, chart, width, legend); + let [px, py, pw, ph] = plan.plot; + if pw <= 0.0 || ph <= 0.0 { + return Hover::default(); + } + let n = categories(chart); + if chart.kind == ChartKind::Scatter { + let mut best: Option<(f32, usize, usize)> = None; + for (s, series) in chart.series.iter().enumerate() { + for (p, (vx, vy)) in series.points.iter().enumerate() { + let sx = px + pw * ((vx - plan.x_lo) / (plan.x_hi - plan.x_lo)) as f32; + let sy = py + ph * (1.0 - ((vy - plan.lo) / (plan.hi - plan.lo)) as f32); + // A fingertip, not a cursor: a wider catch than the desktop's. + let d = (sx - x).hypot(sy - y); + if d < 24.0 && best.is_none_or(|(bd, _, _)| d < bd) { + best = Some((d, s, p)); + } + } + } + return Hover { + category: None, + point: best.map(|(_, s, p)| (s, p)), + }; + } + let along = if chart.horizontal && chart.kind == ChartKind::Bar { + (y - py) / ph + } else { + (x - px) / pw + }; + let ix = if chart.kind == ChartKind::Bar || n <= 1 { + (along * n as f32).floor() + } else { + (along * (n - 1) as f32).round() + }; + Hover { + category: (0.0..n as f32).contains(&ix).then_some(ix as usize), + point: None, + } +} + +/// The whole chart (legend band and plot) at `width`. +pub(crate) fn paint( + ctx: &mut TextCtx, + palette: &Palette, + chart: &Chart, + width: f32, + legend: f32, + hover: Hover, +) -> Paint { + let mut out = Paint::default(); + paint_legend(&mut out, ctx, palette, chart, width, legend, hover); + let plan = plan(ctx, chart, width, legend); + paint_plot(&mut out, ctx, palette, chart, &plan, hover); + out +} + +/// One entry in the legend band: a series (swatch, name, value while +/// scrubbing) or a readout line. +struct LegendItem { + swatch: Option, + name: String, + value: Option, + weight: Weight, + color: Rgba, +} + +impl LegendItem { + fn text(name: String, weight: Weight, color: Rgba) -> Self { + Self { + swatch: None, + name, + value: None, + weight, + color, + } + } +} + +#[allow(clippy::too_many_arguments)] +fn paint_legend( + out: &mut Paint, + ctx: &mut TextCtx, + palette: &Palette, + chart: &Chart, + width: f32, + band: f32, + hover: Hover, +) { + let line = ctx.line_height(TextSize::Sm); + // Readout first (the hovered category, or the hovered point's values), + // then series entries with their values while scrubbing. + let mut items: Vec = Vec::new(); + if let Some(label) = hover.category.and_then(|i| chart.x.get(i)) { + items.push(LegendItem::text( + label.clone(), + Weight::Medium, + palette.text, + )); + } + if let Some((s, p)) = hover.point { + let (x, y) = chart.series[s].points[p]; + let text = format!("{} · {}", chart.x_format.format(x), chart.format.format(y)); + items.push(LegendItem::text(text, Weight::Regular, palette.text)); + } + if show_legend(chart) || hover.category.is_some() { + for series in &chart.series { + items.push(LegendItem { + swatch: Some(palette.color(series.color)), + name: series.name.clone(), + value: hover + .category + .and_then(|i| series.values.get(i).copied().flatten()) + .map(|v| chart.format.format(v)), + weight: Weight::Regular, + color: palette.muted, + }); + } + } + let widths: Vec = items + .iter() + .map(|item| { + item.swatch.map_or(0.0, |_| SWATCH + 6.0) + + measure(ctx, &item.name, sm(item.weight)) + + item + .value + .as_ref() + .map_or(0.0, |v| 6.0 + measure(ctx, v, sm(Weight::Regular))) + }) + .collect(); + let rows = ((band + LEGEND_GAP_Y) / (line + LEGEND_GAP_Y)) + .floor() + .max(1.0) as usize; + for ((x, row), item) in flow(&widths, width).into_iter().zip(&items) { + // What doesn't fit the band is dropped (the band's height is fixed). + if row >= rows { + break; + } + let top = row as f32 * (line + LEGEND_GAP_Y); + let mut at = x; + if let Some(swatch) = item.swatch { + out.rect(at, top + (line - SWATCH) / 2.0, SWATCH, SWATCH, 2.0, swatch); + at += SWATCH + 6.0; + } + at += out.label( + ctx, + &item.name, + sm(item.weight), + item.color, + [at, top], + Align::Start, + Align::Start, + ); + if let Some(value) = &item.value { + out.label( + ctx, + value, + sm(Weight::Regular), + palette.text, + [at + 6.0, top], + Align::Start, + Align::Start, + ); + } + } +} + +fn paint_plot( + out: &mut Paint, + ctx: &mut TextCtx, + palette: &Palette, + chart: &Chart, + plan: &Plan, + hover: Hover, +) { + let [ox, oy, pw, ph] = plan.plot; + if pw <= 4.0 || ph <= 4.0 { + return; + } + let horizontal = chart.kind == ChartKind::Bar && chart.horizontal; + let n = categories(chart).max(1); + let value_t = |v: f64| ((v - plan.lo) / (plan.hi - plan.lo)) as f32; + let vy = |v: f64| oy + ph * (1.0 - value_t(v)); + let vx = |v: f64| ox + pw * value_t(v); + let (grid, muted) = (palette.border, palette.muted); + + // Gridlines and value ticks. + for tick in &plan.ticks { + let label = chart.format.format(*tick); + if horizontal { + let x = vx(*tick); + out.stroke([[x, oy], [x, oy + ph]], false, 1.0, None, grid); + out.label( + ctx, + &label, + xs(), + muted, + [x, oy + ph + AXIS_GAP / 2.0], + Align::Center, + Align::Start, + ); + } else { + let y = vy(*tick); + out.stroke([[ox, y], [ox + pw, y]], false, 1.0, None, grid); + out.label( + ctx, + &label, + xs(), + muted, + [ox - AXIS_GAP, y], + Align::End, + Align::Center, + ); + } + } + + match chart.kind { + ChartKind::Bar => { + let band = if horizontal { ph } else { pw } / n as f32; + if let Some(i) = hover.category { + let start = if horizontal { oy } else { ox } + band * i as f32; + if horizontal { + out.rect(ox, start, pw, band, 4.0, palette.wash); + } else { + out.rect(start, oy, band, ph, 4.0, palette.wash); + } + } + let groups = if chart.stacked { + 1 + } else { + chart.series.len().max(1) + }; + let inner = band * 0.72; + let bar = (inner / groups as f32).max(1.0); + for i in 0..n { + let start = band * i as f32 + (band - inner) / 2.0; + let (mut pos, mut neg) = (0.0f64, 0.0f64); + for (s, series) in chart.series.iter().enumerate() { + let Some(v) = series.values.get(i).copied().flatten() else { + continue; + }; + let (from, to) = if chart.stacked { + if v >= 0.0 { + pos += v; + (pos - v, pos) + } else { + neg += v; + (neg - v, neg) + } + } else { + (0.0f64.clamp(plan.lo, plan.hi), v) + }; + let offset = if chart.stacked { 0.0 } else { bar * s as f32 }; + let gap = if groups > 1 && !chart.stacked { + 1.0 + } else { + 0.0 + }; + let radius = if chart.stacked { 0.0 } else { 2.0 }; + let color = palette.color(series.color); + if horizontal { + let y = oy + start + offset; + let (a, b) = (vx(from), vx(to)); + out.rect(a.min(b), y, (a - b).abs(), bar - gap, radius, color); + } else { + let x = ox + start + offset; + let (a, b) = (vy(from), vy(to)); + out.rect(x, a.min(b), bar - gap, (a - b).abs(), radius, color); + } + } + } + for (i, label) in chart.x.iter().enumerate().take(n) { + if i % plan.label_stride != 0 { + continue; + } + let center = band * (i as f32 + 0.5); + if horizontal { + out.label( + ctx, + label, + xs(), + muted, + [ox - AXIS_GAP, oy + center], + Align::End, + Align::Center, + ); + } else { + out.label( + ctx, + label, + xs(), + muted, + [ox + center, oy + ph + AXIS_GAP / 2.0], + Align::Center, + Align::Start, + ); + } + } + } + ChartKind::Line | ChartKind::Area => { + let x_at = |i: usize| { + if n <= 1 { + ox + pw / 2.0 + } else { + ox + pw * i as f32 / (n - 1) as f32 + } + }; + if let Some(i) = hover.category { + let x = x_at(i); + out.stroke( + [[x, oy], [x, oy + ph]], + false, + 1.0, + None, + palette.border_strong, + ); + } + let base = vy(0.0f64.clamp(plan.lo, plan.hi)); + for series in &chart.series { + let color = palette.color(series.color); + // Gaps split a series into runs. + let mut runs: Vec> = vec![Vec::new()]; + for (i, v) in series.values.iter().enumerate() { + match v { + Some(v) => runs.last_mut().expect("a run").push([x_at(i), vy(*v)]), + None => runs.push(Vec::new()), + } + } + for run in runs.iter().filter(|r| !r.is_empty()) { + if chart.kind == ChartKind::Area && run.len() > 1 { + let mut area = run.clone(); + area.push([run[run.len() - 1][0], base]); + area.push([run[0][0], base]); + out.fill(area, color.alpha(0.22)); + } + if run.len() == 1 { + out.dot(run[0], 2.5, color); + } else { + out.stroke(run.iter().copied(), false, 1.75, None, color); + } + } + if let Some(i) = hover.category + && let Some(Some(v)) = series.values.get(i) + { + out.dot([x_at(i), vy(*v)], 3.5, color); + } + } + for (i, label) in chart.x.iter().enumerate().take(n) { + if i % plan.label_stride != 0 { + continue; + } + let align = if n > 1 && i == 0 { + Align::Start + } else if n > 1 && i == n - 1 { + Align::End + } else { + Align::Center + }; + out.label( + ctx, + label, + xs(), + muted, + [x_at(i), oy + ph + AXIS_GAP / 2.0], + align, + Align::Start, + ); + } + } + ChartKind::Scatter => { + let sx = |x: f64| ox + pw * ((x - plan.x_lo) / (plan.x_hi - plan.x_lo)) as f32; + for tick in &plan.x_ticks { + let x = sx(*tick); + out.stroke([[x, oy], [x, oy + ph]], false, 1.0, None, grid); + out.label( + ctx, + &chart.x_format.format(*tick), + xs(), + muted, + [x, oy + ph + AXIS_GAP / 2.0], + Align::Center, + Align::Start, + ); + } + for (s, series) in chart.series.iter().enumerate() { + let color = palette.color(series.color); + for (ix, (x, y)) in series.points.iter().enumerate() { + let r = if hover.point == Some((s, ix)) { + 5.0 + } else { + 3.0 + }; + out.dot([sx(*x), vy(*y)], r, color); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn domains_include_zero_for_bars_and_land_on_ticks() { + let (lo, hi, ticks) = domain(12.0, 87.0, true); + assert_eq!((lo, hi), (0.0, 100.0)); + assert_eq!(ticks.first(), Some(&0.0)); + assert_eq!(ticks.last(), Some(&100.0)); + let (lo, hi, _) = domain(-3.0, 7.0, false); + assert!(lo <= -3.0 && hi >= 7.0); + } + + #[test] + fn legend_entries_flow_onto_new_rows() { + let at = flow(&[40.0, 40.0, 40.0], 100.0); + assert_eq!(at, vec![(0.0, 0), (54.0, 0), (0.0, 1)]); + } +} diff --git a/crates/mobile/src/visual/layout.rs b/crates/mobile/src/visual/layout.rs new file mode 100644 index 0000000000..6c0dff0e9e --- /dev/null +++ b/crates/mobile/src/visual/layout.rs @@ -0,0 +1,1107 @@ +//! Lays a visual's node tree out at a width: the desktop's stack, grid and +//! table rules (`crates/ui/src/visuals.rs`) as arithmetic, so the platform +//! only paints. Controls are drawn here too (one look on every platform); +//! the platform puts touch targets over them. + +use zeron_visual::{Align, Axis, HandlerId, Justify, Node, NodeKind, TextSize, Weight}; + +use super::chart; +use super::paint::{Face, Mark, Paint, Para, TextCtx}; +use super::palette::Palette; +use super::{ControlKind, SurfaceGesture, VisualControl}; + +/// Control metrics, in points. +pub(crate) mod metrics { + pub const BUTTON_H: f32 = 32.0; + pub const BUTTON_PAD: f32 = 14.0; + pub const TAB_H: f32 = 30.0; + pub const TAB_PAD: f32 = 12.0; + pub const TAB_GAP: f32 = 2.0; + pub const TOGGLE_W: f32 = 36.0; + pub const TOGGLE_H: f32 = 22.0; + pub const TOGGLE_GAP: f32 = 10.0; + pub const SLIDER_TRACK: f32 = 4.0; + pub const SLIDER_THUMB: f32 = 14.0; + pub const TABLE_CELL_PAD: f32 = 12.0; + pub const TABLE_CELL_PAD_Y: f32 = 8.0; +} +use metrics::*; + +/// What a surface draws (every frame it changes). +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum SurfaceNode { + Canvas(zeron_visual::Canvas), + Scene(zeron_visual::SceneNode), + Shader(zeron_visual::ShaderNode), + Chart { + chart: Box, + /// Height of the legend band above the plot. + legend: f32, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SurfaceSpec { + /// Stable across renders (node key or tree path). + pub id: u64, + pub rect: [f32; 4], + pub node: SurfaceNode, +} + +impl SurfaceSpec { + pub fn size(&self) -> (f32, f32) { + (self.rect[2], self.rect[3]) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ActionArg { + None, + Bool(bool), + Text(String), + Slider { min: f64, max: f64, step: f64 }, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Action { + Call(HandlerId, ActionArg), + /// Opened by the platform. + Link, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ControlSpec { + pub control: VisualControl, + pub action: Action, +} + +/// A tree laid out at one width. +#[derive(Default)] +pub(crate) struct Laid { + pub height: f32, + pub paint: Paint, + pub surfaces: Vec, + pub controls: Vec, +} + +#[derive(Clone, Copy)] +struct LaidMark { + paint: Mark, + surfaces: usize, + controls: usize, +} + +impl Laid { + fn mark(&self) -> LaidMark { + LaidMark { + paint: self.paint.mark(), + surfaces: self.surfaces.len(), + controls: self.controls.len(), + } + } + + /// Move everything added between two marks. + fn shift(&mut self, from: LaidMark, to: LaidMark, dx: f32, dy: f32) { + if dx == 0.0 && dy == 0.0 { + return; + } + self.paint.shift_between(from.paint, to.paint, dx, dy); + for s in &mut self.surfaces[from.surfaces..to.surfaces] { + s.rect[0] += dx; + s.rect[1] += dy; + } + for c in &mut self.controls[from.controls..to.controls] { + c.control.x += dx; + c.control.y += dy; + if let ControlKind::Slider { track_x, .. } = &mut c.control.kind { + *track_x += dx; + } + } + } +} + +/// Stable 64-bit id for a node path (FNV-1a). +pub(crate) fn stable_id(s: &str) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in s.as_bytes() { + h ^= u64::from(*b); + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +fn node_key(node: &Node, path: &str) -> String { + match &node.key { + Some(key) => format!("key:{key}"), + None => path.to_owned(), + } +} + +/// Lay `tree` out at `width`. +pub(crate) fn lay_out(ctx: &mut TextCtx, palette: &Palette, tree: &Node, width: f32) -> Laid { + let mut layouter = Layouter { + ctx, + palette, + out: Laid::default(), + }; + let height = layouter.place(tree, "0", 0.0, 0.0, width); + let mut out = layouter.out; + out.height = height; + out +} + +struct Layouter<'a, 'b> { + ctx: &'a mut TextCtx<'b>, + palette: &'a Palette, + out: Laid, +} + +#[derive(Clone, Copy, Debug)] +enum Basis { + Fixed(f32), + Flex, + Shrink(f32), +} + +impl Layouter<'_, '_> { + fn text_para(&mut self, text: &zeron_visual::Text) -> Para { + let base = text + .color + .map(|c| self.palette.color(c)) + .unwrap_or(self.palette.text); + let parts: Vec<(&str, Face, super::Rgba)> = text + .spans + .iter() + .map(|span| { + let face = Face { + size: text.size, + weight: span.weight.unwrap_or(text.weight), + mono: text.mono || span.mono, + italic: text.italic || span.italic, + }; + let color = span.color.map(|c| self.palette.color(c)).unwrap_or(base); + (span.text.as_str(), face, color) + }) + .collect(); + if parts.iter().all(|(s, _, _)| s.is_empty()) { + // An empty text still holds its line. + let face = Face::new(text.size, text.weight); + return self.ctx.para(&[("", face, base)]); + } + self.ctx.para(&parts) + } + + fn plain(&mut self, text: &str, size: TextSize, weight: Weight, color: super::Rgba) -> Para { + self.ctx.line(text, Face::new(size, weight), color) + } + + /// Width a node takes unstretched; `None` fills what it's given. + fn natural(&mut self, node: &Node) -> Option { + if let Some(width) = node.width { + return Some(width); + } + match &node.kind { + NodeKind::Stack(stack) => { + let gaps = stack.gap * stack.children.len().saturating_sub(1) as f32; + let mut total: f32 = 0.0; + for child in &stack.children { + let w = self.natural(child)?; + total = match stack.axis { + Axis::Horizontal => total + w, + Axis::Vertical => total.max(w), + }; + } + Some(match stack.axis { + Axis::Horizontal => total + gaps, + Axis::Vertical => total, + }) + } + NodeKind::Text(text) => Some(self.text_para(text).natural()), + NodeKind::Space(size) => Some(*size), + NodeKind::Stat { + value, + label, + delta, + .. + } => { + let value_w = self + .plain(value, TextSize::Xxl, Weight::Semibold, self.palette.text) + .natural(); + let delta_w = delta.as_deref().filter(|d| !d.is_empty()).map_or(0.0, |d| { + 6.0 + self + .plain(d, TextSize::Sm, Weight::Regular, self.palette.text) + .natural() + }); + let label_w = self + .plain(label, TextSize::Sm, Weight::Regular, self.palette.text) + .natural(); + Some((value_w + delta_w).max(label_w)) + } + NodeKind::Button { label, .. } => Some(self.button_width(label)), + NodeKind::Tabs { options, .. } => { + let widths: Vec = options.iter().map(|o| self.tab_width(o)).collect(); + Some(widths.iter().sum::() + TAB_GAP * widths.len().saturating_sub(1) as f32) + } + NodeKind::Toggle { label, .. } => Some( + TOGGLE_W + + TOGGLE_GAP + + self + .plain(label, TextSize::Body, Weight::Regular, self.palette.text) + .natural(), + ), + NodeKind::Link { label, .. } => Some( + self.plain(label, TextSize::Body, Weight::Regular, self.palette.accent) + .natural(), + ), + NodeKind::Grid { .. } + | NodeKind::Table(_) + | NodeKind::Rule + | NodeKind::Chart(_) + | NodeKind::Canvas(_) + | NodeKind::Scene(_) + | NodeKind::Shader(_) + | NodeKind::Slider { .. } => None, + } + } + + fn button_width(&mut self, label: &str) -> f32 { + self.plain(label, TextSize::Body, Weight::Medium, self.palette.text) + .natural() + + BUTTON_PAD * 2.0 + } + + fn tab_width(&mut self, label: &str) -> f32 { + self.plain(label, TextSize::Body, Weight::Medium, self.palette.text) + .natural() + + TAB_PAD * 2.0 + } + + /// Place `node` with its top-left at (x, y) in `width`; returns its height. + fn place(&mut self, node: &Node, path: &str, x: f32, y: f32, width: f32) -> f32 { + let key = node_key(node, path); + match &node.kind { + NodeKind::Stack(stack) => self.stack(stack, &key, path, x, y, width), + NodeKind::Grid { + columns, + gap, + children, + } => { + let columns = usize::from((*columns).max(1)); + let cell = ((width - gap * (columns - 1) as f32) / columns as f32).max(0.0); + let mut top = y; + for (row, chunk) in children.chunks(columns).enumerate() { + let mut tallest: f32 = 0.0; + for (col, child) in chunk.iter().enumerate() { + let ix = row * columns + col; + let h = self.place( + child, + &format!("{path}.{ix}"), + x + col as f32 * (cell + gap), + top, + cell, + ); + tallest = tallest.max(h); + } + top += tallest + gap; + } + (top - y - gap).max(0.0) + } + NodeKind::Text(text) => { + let para = self.text_para(text); + self.out + .paint + .para(self.ctx, ¶, x, y, width, text.align, text.truncate) + } + NodeKind::Rule => { + self.out + .paint + .rect(x, y, width, 1.0, 0.0, self.palette.border); + 1.0 + } + NodeKind::Space(size) => *size, + NodeKind::Stat { + value, + label, + delta, + tone, + } => self.stat(value, label, delta.as_deref(), *tone, x, y), + NodeKind::Table(table) => self.table(table, path, x, y, width), + NodeKind::Chart(chart) => { + let legend = chart::legend_height(self.ctx, self.palette, chart, width); + let height = legend + chart::LEGEND_GAP + chart.height; + self.surface( + &key, + [x, y, width, height], + SurfaceNode::Chart { + chart: chart.clone(), + legend, + }, + ); + height + } + NodeKind::Canvas(canvas) => { + self.surface( + &key, + [x, y, width, canvas.height], + SurfaceNode::Canvas(canvas.clone()), + ); + canvas.height + } + NodeKind::Scene(scene) => { + self.surface( + &key, + [x, y, width, scene.height], + SurfaceNode::Scene(scene.clone()), + ); + scene.height + } + NodeKind::Shader(shader) => { + self.surface( + &key, + [x, y, width, shader.height], + SurfaceNode::Shader(shader.clone()), + ); + shader.height + } + NodeKind::Button { + label, + on_click, + primary, + } => self.button(&key, label, *on_click, *primary, x, y), + NodeKind::Tabs { + options, + value, + on_change, + } => self.tabs(&key, options, value, *on_change, x, y, width), + NodeKind::Toggle { + label, + value, + on_change, + } => self.toggle(&key, label, *value, *on_change, x, y), + NodeKind::Slider { + label, + value, + min, + max, + step, + on_change, + format, + } => { + let shown = format.format(*value); + self.slider( + &key, + label.as_deref(), + *value, + (*min, *max, *step), + *on_change, + &shown, + x, + y, + width, + ) + } + NodeKind::Link { label, url } => { + let para = self.plain(label, TextSize::Body, Weight::Regular, self.palette.accent); + let w = para.natural().min(width); + let h = self + .out + .paint + .para(self.ctx, ¶, x, y, width, Align::Start, false); + self.control( + &key, + ControlKind::Link { url: url.clone() }, + label, + [x, y, w, h], + Action::Link, + ); + h + } + } + } + + fn surface(&mut self, key: &str, rect: [f32; 4], node: SurfaceNode) { + self.out.surfaces.push(SurfaceSpec { + id: stable_id(key), + rect, + node, + }); + } + + fn control( + &mut self, + key: &str, + kind: ControlKind, + label: &str, + [x, y, w, h]: [f32; 4], + action: Action, + ) { + self.out.controls.push(ControlSpec { + control: VisualControl { + id: stable_id(key), + kind, + x, + y, + w, + h, + label: label.to_owned(), + }, + action, + }); + } + + fn stack( + &mut self, + stack: &zeron_visual::Stack, + key: &str, + path: &str, + x: f32, + y: f32, + width: f32, + ) -> f32 { + // A tappable stack's target goes under its children's (controls + // stack in order), so a button inside it still gets its taps. + let slot = self.out.controls.len(); + let height = match stack.axis { + Axis::Vertical => self.column(stack, path, x, y, width), + Axis::Horizontal if stack.wrap => self.wrapped_row(stack, path, x, y, width), + Axis::Horizontal => self.row(stack, path, x, y, width), + }; + if let Some(handler) = stack.on_click { + self.control( + key, + ControlKind::Tap, + "", + [x, y, width, height], + Action::Call(handler, ActionArg::None), + ); + let tap = self.out.controls.pop().expect("just pushed"); + self.out.controls.insert(slot, tap); + } + height + } + + fn column( + &mut self, + stack: &zeron_visual::Stack, + path: &str, + x: f32, + y: f32, + width: f32, + ) -> f32 { + let mut top = y; + for (ix, child) in stack.children.iter().enumerate() { + if ix > 0 { + top += stack.gap; + } + // Controls keep their own width; content stretches unless the + // stack centers or ends it. + let intrinsic = matches!( + child.kind, + NodeKind::Button { .. } + | NodeKind::Toggle { .. } + | NodeKind::Link { .. } + | NodeKind::Space(_) + | NodeKind::Stat { .. } + ); + let w = match child.width { + Some(w) => w.min(width), + None if stack.align == Align::Start && !intrinsic => width, + None => self.natural(child).map_or(width, |n| n.min(width)), + }; + let offset = match stack.align { + Align::Start => 0.0, + Align::Center => (width - w) / 2.0, + Align::End => width - w, + }; + top += self.place(child, &format!("{path}.{ix}"), x + offset, top, w); + } + top - y + } + + fn bases(&mut self, stack: &zeron_visual::Stack) -> Vec { + stack + .children + .iter() + .map(|child| match (child.width, child.grow) { + (Some(w), _) => Basis::Fixed(w), + (None, true) => Basis::Flex, + (None, false) => match self.natural(child) { + None => Basis::Flex, + Some(w) => Basis::Shrink(w), + }, + }) + .collect() + } + + /// Place a line of children at their widths, aligned across the line; + /// returns the line's height. + fn line( + &mut self, + stack: &zeron_visual::Stack, + path: &str, + items: &[(usize, f32, f32)], + y: f32, + ) -> f32 { + let mut placed = Vec::with_capacity(items.len()); + for (ix, left, w) in items { + let start = self.out.mark(); + let h = self.place(&stack.children[*ix], &format!("{path}.{ix}"), *left, y, *w); + placed.push((start, self.out.mark(), h)); + } + let tallest = placed.iter().map(|(_, _, h)| *h).fold(0.0, f32::max); + for (from, to, h) in placed { + let dy = match stack.align { + Align::Start => 0.0, + Align::Center => (tallest - h) / 2.0, + Align::End => tallest - h, + }; + self.out.shift(from, to, 0.0, dy); + } + tallest + } + + /// Lefts for `widths` from x, with justify spreading `leftover`. + fn justify(stack: &zeron_visual::Stack, x: f32, widths: &[f32], leftover: f32) -> Vec { + let n = widths.len(); + let (mut at, between) = match stack.justify { + Justify::Start => (x, 0.0), + Justify::Center => (x + leftover / 2.0, 0.0), + Justify::End => (x + leftover, 0.0), + Justify::Between if n > 1 => (x, leftover / (n - 1) as f32), + Justify::Between => (x, 0.0), + }; + widths + .iter() + .map(|w| { + let left = at; + at += w + stack.gap + between; + left + }) + .collect() + } + + fn row(&mut self, stack: &zeron_visual::Stack, path: &str, x: f32, y: f32, width: f32) -> f32 { + let n = stack.children.len(); + if n == 0 { + return 0.0; + } + let bases = self.bases(stack); + let gaps = stack.gap * (n - 1) as f32; + let used: f32 = bases + .iter() + .map(|b| match b { + Basis::Fixed(w) | Basis::Shrink(w) => *w, + Basis::Flex => 0.0, + }) + .sum(); + let free = width - gaps - used; + let flexes = bases.iter().filter(|b| matches!(b, Basis::Flex)).count(); + let shrinkable: f32 = bases + .iter() + .map(|b| if let Basis::Shrink(w) = b { *w } else { 0.0 }) + .sum(); + let widths: Vec = bases + .iter() + .map(|b| match *b { + Basis::Fixed(w) => w, + Basis::Flex if free > 0.0 => free / flexes as f32, + Basis::Flex => 0.0, + Basis::Shrink(w) if free < 0.0 && shrinkable > 0.0 => { + (w + free * w / shrinkable).max(0.0) + } + Basis::Shrink(w) => w, + }) + .collect(); + let leftover = if flexes == 0 { free.max(0.0) } else { 0.0 }; + let lefts = Self::justify(stack, x, &widths, leftover); + let items: Vec<(usize, f32, f32)> = (0..n).map(|i| (i, lefts[i], widths[i])).collect(); + self.line(stack, path, &items, y) + } + + fn wrapped_row( + &mut self, + stack: &zeron_visual::Stack, + path: &str, + x: f32, + y: f32, + width: f32, + ) -> f32 { + let bases = self.bases(stack); + let mut lines: Vec> = vec![Vec::new()]; + let mut cursor = 0.0; + for (ix, basis) in bases.iter().enumerate() { + let (w, flex) = match *basis { + Basis::Fixed(w) => (w.min(width), false), + Basis::Shrink(w) => (w.min(width), false), + Basis::Flex => ( + self.natural(&stack.children[ix]) + .unwrap_or(width) + .min(width), + true, + ), + }; + let line = lines.last_mut().expect("one line"); + let next = if line.is_empty() { + w + } else { + cursor + stack.gap + w + }; + if !line.is_empty() && next > width + 0.5 { + lines.push(vec![(ix, w, flex)]); + cursor = w; + } else { + line.push((ix, w, flex)); + cursor = next; + } + } + let mut top = y; + for (n, line) in lines.iter().enumerate() { + if line.is_empty() { + continue; + } + if n > 0 { + top += stack.gap; + } + let used: f32 = + line.iter().map(|(_, w, _)| w).sum::() + stack.gap * (line.len() - 1) as f32; + let free = (width - used).max(0.0); + let flexes = line.iter().filter(|(_, _, flex)| *flex).count(); + let widths: Vec = line + .iter() + .map(|(_, w, flex)| { + if *flex && flexes > 0 { + w + free / flexes as f32 + } else { + *w + } + }) + .collect(); + let leftover = if flexes == 0 { free } else { 0.0 }; + let lefts = Self::justify(stack, x, &widths, leftover); + let items: Vec<(usize, f32, f32)> = line + .iter() + .enumerate() + .map(|(i, (ix, _, _))| (*ix, lefts[i], widths[i])) + .collect(); + top += self.line(stack, path, &items, top); + } + top - y + } + + /// A headline figure over a short label: plain type, no tile. + fn stat( + &mut self, + value: &str, + label: &str, + delta: Option<&str>, + tone: Option, + x: f32, + y: f32, + ) -> f32 { + let tone = tone.map(|c| self.palette.color(c)); + let delta = delta.filter(|d| !d.is_empty()); + // A tone marks the change, not the figure; a figure with no change + // takes the tone itself. + let value_color = tone + .filter(|_| delta.is_none()) + .unwrap_or(self.palette.text); + let figure = self.plain(value, TextSize::Xxl, Weight::Semibold, value_color); + let value_w = figure.natural(); + let mut height = self.out.paint.para( + self.ctx, + &figure, + x, + y, + value_w.max(1.0), + Align::Start, + true, + ); + if let Some(delta) = delta { + let change = self.plain( + delta, + TextSize::Sm, + Weight::Regular, + tone.unwrap_or(self.palette.muted), + ); + // Baseline-aligned with the figure. + let top = y + figure.base - change.base; + self.out.paint.para( + self.ctx, + &change, + x + value_w + 6.0, + top, + change.natural().max(1.0), + Align::Start, + true, + ); + } + if !label.is_empty() { + let caption = self.plain(label, TextSize::Sm, Weight::Regular, self.palette.muted); + height += 2.0; + height += self.out.paint.para( + self.ctx, + &caption, + x, + y + height, + caption.natural().max(1.0), + Align::Start, + false, + ); + } + height + } + + /// The markdown table's frameless treatment: a bold header and hairlines + /// between rows, no outer box or fills. + fn table( + &mut self, + table: &zeron_visual::TableNode, + path: &str, + x: f32, + y: f32, + width: f32, + ) -> f32 { + let columns = table + .columns + .len() + .max(table.rows.iter().map(Vec::len).max().unwrap_or(0)); + if columns == 0 { + return 0.0; + } + let column = |c: usize| table.columns.get(c).cloned().unwrap_or_default(); + let fixed: f32 = (0..columns).filter_map(|c| column(c).width).sum(); + let flexes = (0..columns).filter(|c| column(*c).width.is_none()).count(); + let share = if flexes > 0 { + ((width - fixed) / flexes as f32).max(0.0) + } else { + 0.0 + }; + let widths: Vec = (0..columns) + .map(|c| column(c).width.unwrap_or(share)) + .collect(); + let lefts: Vec = widths + .iter() + .scan(x, |at, w| { + let left = *at; + *at += w; + Some(left) + }) + .collect(); + let hairline = self.palette.text.alpha(0.10); + let mut top = y; + // One row: each cell's content aligned in its padded box. + let row = |this: &mut Self, + top: f32, + cells: &mut dyn FnMut(&mut Self, usize, f32, f32, f32) -> f32| + -> f32 { + let mut tallest: f32 = 0.0; + for c in 0..columns { + let inner = (widths[c] - TABLE_CELL_PAD * 2.0).max(0.0); + tallest = tallest.max(cells( + this, + c, + lefts[c] + TABLE_CELL_PAD, + top + TABLE_CELL_PAD_Y, + inner, + )); + } + tallest + TABLE_CELL_PAD_Y * 2.0 + }; + if table.columns.iter().any(|c| !c.label.is_empty()) { + top += row(self, top, &mut |this, c, cx, cy, inner| { + let col = column(c); + let para = this.plain(&col.label, TextSize::Body, Weight::Bold, this.palette.text); + let w = para.natural().min(inner); + let offset = align_offset(col.align, inner, w); + this.out.paint.para( + this.ctx, + ¶, + cx + offset, + cy, + w.max(1.0), + Align::Start, + false, + ) + }); + } + for (r, cells) in table.rows.iter().enumerate() { + if r > 0 || !table.columns.is_empty() { + self.out.paint.rect(x, top, width, 1.0, 0.0, hairline); + top += 1.0; + } + top += row(self, top, &mut |this, c, cx, cy, inner| { + let Some(node) = cells.get(c) else { + return 0.0; + }; + let w = this.natural(node).map_or(inner, |n| n.min(inner)); + let offset = align_offset(column(c).align, inner, w); + this.place(node, &format!("{path}.{r}.{c}"), cx + offset, cy, w) + }); + } + top - y + } + + fn button( + &mut self, + key: &str, + label: &str, + handler: HandlerId, + primary: bool, + x: f32, + y: f32, + ) -> f32 { + let w = self.button_width(label); + let (fill, ink) = if primary { + (self.palette.accent, self.palette.on_accent) + } else { + (self.palette.wash, self.palette.text) + }; + self.out.paint.rect(x, y, w, BUTTON_H, 8.0, fill); + let para = self.plain(label, TextSize::Body, Weight::Medium, ink); + self.out.paint.para( + self.ctx, + ¶, + x + BUTTON_PAD, + y + (BUTTON_H - para.lh) / 2.0, + w, + Align::Start, + true, + ); + self.control( + key, + ControlKind::Button { primary }, + label, + [x, y, w, BUTTON_H], + Action::Call(handler, ActionArg::None), + ); + BUTTON_H + } + + #[allow(clippy::too_many_arguments)] + fn tabs( + &mut self, + key: &str, + options: &[String], + value: &str, + handler: HandlerId, + x: f32, + y: f32, + width: f32, + ) -> f32 { + let (mut cx, mut top) = (x, y); + for (ix, option) in options.iter().enumerate() { + let w = self.tab_width(option); + if cx > x && cx + w > x + width + 0.5 { + cx = x; + top += TAB_H + TAB_GAP; + } + let selected = option == value; + if selected { + self.out + .paint + .rect(cx, top, w, TAB_H, 8.0, self.palette.accent.alpha(0.14)); + } + let (weight, ink) = if selected { + (Weight::Semibold, self.palette.accent) + } else { + (Weight::Medium, self.palette.muted) + }; + let para = self.plain(option, TextSize::Body, weight, ink); + self.out.paint.para( + self.ctx, + ¶, + cx + TAB_PAD, + top + (TAB_H - para.lh) / 2.0, + w, + Align::Start, + true, + ); + self.control( + &format!("{key}#tab{ix}"), + ControlKind::Tab { selected }, + option, + [cx, top, w, TAB_H], + Action::Call(handler, ActionArg::Text(option.clone())), + ); + cx += w + TAB_GAP; + } + if options.is_empty() { + 0.0 + } else { + top + TAB_H - y + } + } + + fn toggle( + &mut self, + key: &str, + label: &str, + on: bool, + handler: HandlerId, + x: f32, + y: f32, + ) -> f32 { + let para = self.plain(label, TextSize::Body, Weight::Regular, self.palette.text); + let h = para.lh.max(TOGGLE_H); + let track_y = y + (h - TOGGLE_H) / 2.0; + let track = if on { + self.palette.accent + } else { + self.palette.text.alpha(0.16) + }; + self.out + .paint + .rect(x, track_y, TOGGLE_W, TOGGLE_H, TOGGLE_H / 2.0, track); + let knob = TOGGLE_H - 4.0; + let knob_x = if on { + x + TOGGLE_W - knob - 2.0 + } else { + x + 2.0 + }; + self.out.paint.rect( + knob_x, + track_y + 2.0, + knob, + knob, + knob / 2.0, + self.palette.on_accent, + ); + let text_x = x + TOGGLE_W + TOGGLE_GAP; + let text_w = para.natural(); + self.out.paint.para( + self.ctx, + ¶, + text_x, + y + (h - para.lh) / 2.0, + text_w.max(1.0), + Align::Start, + true, + ); + self.control( + key, + ControlKind::Toggle { on }, + label, + [x, y, TOGGLE_W + TOGGLE_GAP + text_w, h], + Action::Call(handler, ActionArg::Bool(!on)), + ); + h + } + + #[allow(clippy::too_many_arguments)] + fn slider( + &mut self, + key: &str, + label: Option<&str>, + value: f64, + (min, max, step): (f64, f64, f64), + handler: HandlerId, + shown: &str, + x: f32, + y: f32, + width: f32, + ) -> f32 { + let caption = self.plain( + label.unwrap_or(""), + TextSize::Sm, + Weight::Regular, + self.palette.muted, + ); + let readout = self.plain(shown, TextSize::Sm, Weight::Regular, self.palette.text); + let readout_w = readout.natural(); + self.out.paint.para( + self.ctx, + &caption, + x, + y, + (width - readout_w - 12.0).max(1.0), + Align::Start, + true, + ); + self.out.paint.para( + self.ctx, + &readout, + x + width - readout_w, + y, + readout_w.max(1.0), + Align::Start, + true, + ); + let row = caption.lh.max(readout.lh); + let track_top = y + row + 4.0; + let track_h = SLIDER_THUMB + 4.0; + let mid = track_top + track_h / 2.0; + let left = x + SLIDER_THUMB / 2.0; + let span = (width - SLIDER_THUMB).max(1.0); + let fraction = if max > min { + ((value - min) / (max - min)).clamp(0.0, 1.0) as f32 + } else { + 0.0 + }; + let at = left + span * fraction; + let half = SLIDER_TRACK / 2.0; + self.out.paint.rect( + left, + mid - half, + span, + SLIDER_TRACK, + half, + self.palette.border, + ); + self.out.paint.rect( + left, + mid - half, + at - left, + SLIDER_TRACK, + half, + self.palette.accent, + ); + self.out + .paint + .dot([at, mid], SLIDER_THUMB / 2.0, self.palette.text); + self.control( + key, + ControlKind::Slider { + value, + min, + max, + step, + fraction, + track_x: left, + track_w: span, + shown: shown.to_owned(), + }, + label.unwrap_or(""), + [x, track_top, width, track_h], + Action::Call(handler, ActionArg::Slider { min, max, step }), + ); + track_top + track_h - y + } +} + +fn align_offset(align: Align, room: f32, w: f32) -> f32 { + match align { + Align::Start => 0.0, + Align::Center => ((room - w) / 2.0).max(0.0), + Align::End => (room - w).max(0.0), + } +} + +/// How a surface takes touches (canvas pointer use is learned by drawing). +pub(crate) fn gesture(node: &SurfaceNode, reads_pointer: bool) -> SurfaceGesture { + match node { + SurfaceNode::Scene(scene) if scene.orbit => SurfaceGesture::Drag, + SurfaceNode::Scene(_) => SurfaceGesture::Tap, + SurfaceNode::Shader(shader) + if shader.orbit + || shader.code.contains("iMouse") + || shader.buffers.iter().any(|b| b.contains("iMouse")) => + { + SurfaceGesture::Drag + } + SurfaceNode::Shader(_) => SurfaceGesture::None, + SurfaceNode::Canvas(_) if reads_pointer => SurfaceGesture::Drag, + SurfaceNode::Canvas(canvas) if canvas.on_click.is_some() => SurfaceGesture::Tap, + SurfaceNode::Canvas(_) => SurfaceGesture::None, + SurfaceNode::Chart { .. } => SurfaceGesture::HorizontalDrag, + } +} diff --git a/crates/mobile/src/visual/mod.rs b/crates/mobile/src/visual/mod.rs new file mode 100644 index 0000000000..50e090c839 --- /dev/null +++ b/crates/mobile/src/visual/mod.rs @@ -0,0 +1,1468 @@ +//! Agent visuals (`show_visual` Luau scripts) on mobile. +//! +//! Each visual in a transcript gets one worker thread that owns its Luau +//! runtime, lays its node tree out (text measured with the bundled faces), +//! draws canvases and charts as vector paint, and renders scenes and shaders +//! on the GPU. Nothing here runs on the platform's UI thread: the platform +//! receives finished display lists and frames through [`VisualListener`], +//! paints them, and forwards touches. The transcript learns each visual's +//! height from its worker, so rows are measured before they're painted. +//! +//! Semantics follow the desktop (`crates/ui/src/visuals.rs`); a touch +//! stands in for the pointer. + +mod chart; +mod layout; +mod paint; +mod palette; + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use zeron_text::WidthCache; +use zeron_visual::{DrawCmd, HandlerArg, HandlerId, Node, Visual}; + +use crate::layout::style::Typography; +use crate::layout::{StyleDesc, TextSystem}; +use layout::{Action, ActionArg, Laid, SurfaceNode, SurfaceSpec}; +use paint::{Face, Paint, TextCtx}; +use palette::Palette; + +// ---- FFI ---------------------------------------------------------------------- + +/// sRGB, straight alpha, 0–1. +#[derive(Debug, Clone, Copy, PartialEq, Default, uniffi::Record)] +pub struct Rgba { + pub r: f32, + pub g: f32, + pub b: f32, + pub a: f32, +} + +/// `text[start..start+len]` (UTF-16) in `style`, its origin at (x, baseline). +#[derive(Debug, Clone, PartialEq, uniffi::Record)] +pub struct VisualRun { + pub start: u32, + pub len: u32, + pub x: f32, + pub baseline: f32, + pub width: f32, + pub style: u16, + pub color: Rgba, +} + +#[derive(Debug, Clone, PartialEq, uniffi::Enum)] +pub enum VisualShape { + Rect { + x: f32, + y: f32, + w: f32, + h: f32, + radius: f32, + color: Rgba, + }, + /// A polyline: `points` holds x, y pairs. + Path { + points: Vec, + closed: bool, + fill: Option, + stroke: Option, + line_width: f32, + /// Dash and gap length. + dash: Option, + }, +} + +/// One layer of vector paint: shapes, then text over them. +#[derive(Debug, Clone, PartialEq, Default, uniffi::Record)] +pub struct VisualPaint { + pub text: String, + pub runs: Vec, + pub shapes: Vec, +} + +/// How a surface takes touches. +#[derive(Debug, Clone, Copy, PartialEq, Eq, uniffi::Enum)] +pub enum SurfaceGesture { + /// Not interactive: drags scroll the transcript. + None, + /// Taps only. + Tap, + /// Takes drags in any direction (orbits, pointer-driven drawings). + Drag, + /// Takes mostly-horizontal drags (scrubbing a chart); vertical ones + /// scroll. + HorizontalDrag, +} + +/// A region the worker redraws on its own (canvas, chart, scene, shader). +#[derive(Debug, Clone, PartialEq, uniffi::Record)] +pub struct VisualSurface { + /// Stable across renders: frames and touches address it. + pub id: u64, + pub x: f32, + pub y: f32, + pub w: f32, + pub h: f32, + pub gesture: SurfaceGesture, +} + +#[derive(Debug, Clone, PartialEq, uniffi::Enum)] +pub enum ControlKind { + Button { + primary: bool, + }, + Tab { + selected: bool, + }, + Toggle { + on: bool, + }, + /// The track runs from `track_x` for `track_w` points. + Slider { + value: f64, + min: f64, + max: f64, + step: f64, + fraction: f32, + track_x: f32, + track_w: f32, + shown: String, + }, + /// Opened by the platform. + Link { + url: String, + }, + /// A tappable stack. + Tap, +} + +/// A touch target over painted control art. +#[derive(Debug, Clone, PartialEq, uniffi::Record)] +pub struct VisualControl { + /// Stable across renders. + pub id: u64, + pub kind: ControlKind, + pub x: f32, + pub y: f32, + pub w: f32, + pub h: f32, + /// Accessibility label. + pub label: String, +} + +/// A visual laid out at one width. +#[derive(Debug, Clone, PartialEq, uniffi::Record)] +pub struct VisualDisplay { + pub revision: u64, + pub width: f32, + pub height: f32, + pub dark: bool, + pub paint: VisualPaint, + pub surfaces: Vec, + pub controls: Vec, + /// Style id → face and size, for every run so far. + pub styles: Vec, +} + +/// Premultiplied BGRA8, top row first (CoreGraphics' native +/// `byteOrder32Little | premultipliedFirst`). +#[derive(Debug, Clone, PartialEq, uniffi::Record)] +pub struct FrameImage { + pub width: u32, + pub height: u32, + pub bgra: Vec, +} + +/// A surface's latest picture: an image (scenes, shaders) under vector +/// paint (canvas drawings, charts, scene labels), in surface coordinates. +#[derive(Debug, Clone, PartialEq, uniffi::Record)] +pub struct SurfaceFrame { + pub surface: u64, + pub image: Option, + pub paint: VisualPaint, + pub styles: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, uniffi::Enum)] +pub enum PointerPhase { + Down, + Move, + Up, + Cancel, +} + +/// Where a visual's output goes (called on its worker thread). +#[uniffi::export(with_foreign)] +pub trait VisualListener: Send + Sync { + fn display(&self, display: VisualDisplay); + fn frame(&self, frame: SurfaceFrame); +} + +// ---- transcript side -------------------------------------------------------- + +/// A visual row's link to its worker: the height it measured. +pub(crate) struct VisualSlot { + pub id: String, + pub path: String, + pub title: String, + measured: Mutex>, + epoch: AtomicU64, +} + +impl VisualSlot { + pub fn new(id: &str, path: &str, title: &str) -> Arc { + Arc::new(Self { + id: id.to_owned(), + path: path.to_owned(), + title: title.to_owned(), + measured: Mutex::new(None), + epoch: AtomicU64::new(0), + }) + } + + /// The measured height (at the latest width the worker laid out). + pub fn height(&self) -> Option { + self.snapshot().1 + } + + /// The epoch and the height it belongs to, read together. + pub fn snapshot(&self) -> (u64, Option) { + let measured = self.measured.lock().unwrap(); + (self.epoch.load(Ordering::Relaxed), measured.map(|(_, h)| h)) + } + + fn set(&self, width: f32, height: f32) -> bool { + let mut measured = self.measured.lock().unwrap(); + if measured.is_some_and(|(w, h)| w == width && (h - height).abs() < 0.01) { + return false; + } + let changed = measured.is_none_or(|(_, h)| (h - height).abs() >= 0.01); + *measured = Some((width, height)); + if changed { + self.epoch.fetch_add(1, Ordering::Relaxed); + } + changed + } +} + +/// Reads a file beside the chat (the script, its manifest and assets). +type Fetch = Arc Option> + Send + Sync>; + +#[derive(Clone)] +struct Source { + fetch: Fetch, + pixel_scale: f32, +} + +/// One transcript's visuals: their workers, by visual id. +pub(crate) struct VisualHub { + text: Arc, + /// Set once the transcript closes: no new workers after that. + closed: std::sync::atomic::AtomicBool, + source: Mutex>, + sessions: Mutex>>, + /// Ask the transcript to remeasure a row (its key). + changed: Arc, +} + +impl VisualHub { + pub fn new(text: Arc, changed: Arc) -> Self { + Self { + text, + closed: std::sync::atomic::AtomicBool::new(false), + source: Mutex::new(None), + sessions: Mutex::new(HashMap::new()), + changed, + } + } + + /// Where scripts load from: the chat's host device. + pub fn set_source(&self, client: zeron_client::Client, device: String, pixel_scale: f32) { + let fetch: Fetch = Arc::new(move |path: &str| { + let (client, device, path) = (client.clone(), device.clone(), path.to_owned()); + block_on(zeron_client::runtime::run(async move { + client.read_attachment(&device, &path).await + })) + .ok() + .map(|bytes| bytes.as_ref().clone()) + }); + *self.source.lock().unwrap() = Some(Source { fetch, pixel_scale }); + } + + #[cfg(test)] + pub(crate) fn set_fetch(&self, fetch: Fetch, pixel_scale: f32) { + *self.source.lock().unwrap() = Some(Source { fetch, pixel_scale }); + } + + /// Start (or keep) a visual's worker and lay it out at `width`. + pub fn ensure(&self, slot: &Arc, row_key: u64, width: f32, text_scale: f32) { + if self.closed.load(Ordering::Acquire) { + return; + } + let Some(source) = self.source.lock().unwrap().clone() else { + return; + }; + let session = self + .sessions + .lock() + .unwrap() + .entry(slot.id.clone()) + .or_insert_with(|| { + VisualSession::spawn( + &self.text, + slot.clone(), + row_key, + source, + self.changed.clone(), + ) + }) + .clone(); + session.send(Cmd::Layout { + width, + scale: text_scale, + }); + } + + pub fn session(&self, id: &str) -> Option> { + self.sessions.lock().unwrap().get(id).cloned() + } + + pub fn shutdown(&self) { + self.closed.store(true, Ordering::Release); + for (_, session) in self.sessions.lock().unwrap().drain() { + session.send(Cmd::Shutdown); + } + } +} + +impl Drop for VisualHub { + fn drop(&mut self) { + self.shutdown(); + } +} + +// ---- session ------------------------------------------------------------------ + +enum Cmd { + Layout { + width: f32, + scale: f32, + }, + Attach { + listener: Arc, + dark: bool, + reduce_motion: bool, + }, + Detach, + Appearance { + dark: bool, + }, + Pointer { + surface: u64, + phase: PointerPhase, + x: f32, + y: f32, + }, + Activate { + control: u64, + }, + Slide { + control: u64, + fraction: f32, + }, + Shutdown, +} + +/// One visual: the handle the platform paints and touches through. +#[derive(uniffi::Object)] +pub struct VisualSession { + tx: Mutex>, + title: String, +} + +impl VisualSession { + fn spawn( + text: &Arc, + slot: Arc, + row_key: u64, + source: Source, + changed: Arc, + ) -> Arc { + let (tx, rx) = mpsc::channel(); + let title = slot.title.clone(); + let text = text.clone(); + std::thread::Builder::new() + .name("zeron-visual".into()) + .spawn(move || Worker::new(&text, rx, slot, row_key, source, changed).run()) + .expect("spawn visual thread"); + Arc::new(Self { + tx: Mutex::new(tx), + title, + }) + } + + fn send(&self, cmd: Cmd) { + let _ = self.tx.lock().unwrap().send(cmd); + } +} + +#[uniffi::export] +impl VisualSession { + /// Start delivering displays and frames (replacing any listener). + pub fn attach(&self, listener: Arc, dark: bool, reduce_motion: bool) { + self.send(Cmd::Attach { + listener, + dark, + reduce_motion, + }); + } + + /// Stop delivering (offscreen); animation pauses. + pub fn detach(&self) { + self.send(Cmd::Detach); + } + + pub fn set_appearance(&self, dark: bool) { + self.send(Cmd::Appearance { dark }); + } + + /// A touch on a surface, in its coordinates. + pub fn pointer(&self, surface: u64, phase: PointerPhase, x: f32, y: f32) { + self.send(Cmd::Pointer { + surface, + phase, + x, + y, + }); + } + + /// Tap a button, tab, toggle or tappable stack. + pub fn activate(&self, control: u64) { + self.send(Cmd::Activate { control }); + } + + /// Move a slider to `fraction` (0–1) of its track. + pub fn slide(&self, control: u64, fraction: f32) { + self.send(Cmd::Slide { control, fraction }); + } + + pub fn title(&self) -> String { + self.title.clone() + } +} + +// ---- worker ----------------------------------------------------------------------- + +/// Frames for animated surfaces come this often at most. +const FRAME: Duration = Duration::from_micros(16_667); +/// GPU surfaces render at most this dense (3× phones would quadruple the +/// pixels for little visible gain, and every frame crosses to the platform). +const MAX_GPU_SCALE: f32 = 2.0; + +enum Load { + Loading, + Ready, + Failed(String), +} + +struct Orbit { + yaw: f32, + pitch: f32, + spin_epoch: Instant, + /// Touch start and the camera then. + drag: Option<([f32; 2], f32, f32)>, +} + +impl Orbit { + fn yaw(&self, spin: f32) -> f32 { + match self.drag { + Some(_) => self.yaw, + None => self.yaw + spin * self.spin_epoch.elapsed().as_secs_f32(), + } + } +} + +/// One surface's state between frames. +struct SurfaceState { + spec: SurfaceSpec, + dirty: bool, + /// Touch position and whether it's down. + pointer: Option<[f32; 2]>, + down: bool, + press: Option<[f32; 2]>, + moved: bool, + reads_pointer: bool, + orbit: Option, + shader: Option, + shader_frame: u32, + shader_time: f64, + scene: Option<(u64, zeron_visual::Scene)>, + hints: Vec<([f32; 2], f32, String)>, + hint: Option, + hover: chart::Hover, +} + +impl SurfaceState { + fn new(spec: SurfaceSpec) -> Self { + let orbit = match &spec.node { + SurfaceNode::Scene(s) => Some((s.yaw, s.pitch)), + SurfaceNode::Shader(s) => Some((s.yaw, s.pitch)), + _ => None, + } + .map(|(yaw, pitch)| Orbit { + yaw, + pitch, + spin_epoch: Instant::now(), + drag: None, + }); + Self { + spec, + dirty: true, + pointer: None, + down: false, + press: None, + moved: false, + reads_pointer: false, + orbit, + shader: None, + shader_frame: 0, + shader_time: 0.0, + scene: None, + hints: Vec::new(), + hint: None, + hover: chart::Hover::default(), + } + } +} + +struct Worker { + rx: Receiver, + slot: Arc, + row_key: u64, + changed: Arc, + source: Source, + typo: Typography, + cache: WidthCache, + load: Load, + visual: Option, + tree: Option, + /// Bumped on every tree render: scene drawings key on it. + generation: u64, + error: Option, + width: f32, + scale: f32, + palette: Palette, + reduce_motion: bool, + laid: Laid, + revision: u64, + display_dirty: bool, + listener: Option>, + surfaces: HashMap, + started: Instant, + last_frame: Instant, + /// When to try loading again after the host couldn't be reached, and + /// how many tries so far. + retry: Option<(Instant, u32)>, +} + +impl Worker { + fn new( + text: &TextSystem, + rx: Receiver, + slot: Arc, + row_key: u64, + source: Source, + changed: Arc, + ) -> Self { + Self { + rx, + slot, + row_key, + changed, + source, + typo: crate::layout::typography(text), + cache: WidthCache::new(), + load: Load::Loading, + visual: None, + tree: None, + generation: 0, + error: None, + width: 0.0, + scale: 1.0, + palette: Palette::new(false), + reduce_motion: false, + laid: Laid::default(), + revision: 0, + display_dirty: false, + listener: None, + surfaces: HashMap::new(), + started: Instant::now(), + last_frame: Instant::now(), + retry: None, + } + } + + fn run(mut self) { + // Measure the loading line first, so the row has a height while the + // script and its files arrive. + let queued: Vec = self.rx.try_iter().collect(); + if !self.drain(queued) { + return; + } + self.relayout(); + self.load(); + self.relayout(); + loop { + if self.retry.is_some_and(|(at, _)| Instant::now() >= at) { + self.load(); + self.relayout(); + } + self.publish(); + let first = match self.next_frame() { + Some(wait) => match self.rx.recv_timeout(wait) { + Ok(cmd) => Some(cmd), + Err(RecvTimeoutError::Timeout) => None, + Err(RecvTimeoutError::Disconnected) => return, + }, + None => match self.rx.recv() { + Ok(cmd) => Some(cmd), + Err(_) => return, + }, + }; + let cmds: Vec = first.into_iter().chain(self.rx.try_iter()).collect(); + if !self.drain(cmds) { + return; + } + } + } + + /// Apply commands; false on shutdown. + fn drain(&mut self, cmds: Vec) -> bool { + let mut relayout = false; + for cmd in cmds { + match cmd { + Cmd::Shutdown => return false, + Cmd::Layout { width, scale } => { + if (width - self.width).abs() > 0.01 + || (scale - self.scale).abs() > f32::EPSILON + { + if (scale - self.typo.scale).abs() > f32::EPSILON { + self.typo.scale = scale; + self.cache = WidthCache::new(); + } + self.width = width; + self.scale = scale; + relayout = true; + } + } + Cmd::Attach { + listener, + dark, + reduce_motion, + } => { + self.listener = Some(listener); + self.reduce_motion = reduce_motion; + if self.palette.dark != dark { + self.palette = Palette::new(dark); + relayout = true; + } + self.display_dirty = true; + for state in self.surfaces.values_mut() { + state.dirty = true; + } + } + Cmd::Detach => { + self.listener = None; + } + Cmd::Appearance { dark } => { + if self.palette.dark != dark { + self.palette = Palette::new(dark); + relayout = true; + } + } + Cmd::Pointer { + surface, + phase, + x, + y, + // A handler re-renders the tree; the next command must find + // its controls (and handler ids) in that tree. + } => { + if self.pointer(surface, phase, [x, y]) { + self.relayout(); + } + } + Cmd::Activate { control } => { + if self.activate(control) { + self.relayout(); + } + } + Cmd::Slide { control, fraction } => { + if self.slide(control, fraction) { + self.relayout(); + } + } + } + } + if relayout { + self.relayout(); + } + true + } + + // ---- loading --------------------------------------------------------------- + + fn fetch(&self, path: String) -> Option> { + (self.source.fetch)(&path) + } + + fn load(&mut self) { + let path = self.slot.path.clone(); + let Some(script) = self + .fetch(path.clone()) + .and_then(|b| String::from_utf8(b).ok()) + else { + // The host may be briefly out of reach: try again, backing off. + let tries = self.retry.map_or(0, |(_, tries)| tries) + 1; + let wait = Duration::from_secs([2, 5, 15, 30][(tries as usize - 1).min(3)]); + self.retry = Some((Instant::now() + wait, tries)); + self.load = Load::Failed("Visual unavailable".into()); + return; + }; + self.retry = None; + // Its files sit beside it, listed in `{id}-assets.json` (absent for + // a visual without any). + let mut files = HashMap::new(); + if let Some(stem) = path.strip_suffix("-visual.luau") + && let Some(names) = self + .fetch(format!("{stem}-assets.json")) + .and_then(|bytes| serde_json::from_slice::>(&bytes).ok()) + { + for name in names + .into_iter() + .filter(|name| zeron_visual::assets::valid_name(name)) + .take(zeron_visual::assets::MAX_ASSETS) + { + if let Some(bytes) = self.fetch(format!("{stem}-asset-{name}")) { + files.insert(name, Arc::<[u8]>::from(bytes)); + } + } + } + // Seeded like the engine's check, so the reader sees the same data. + let seed = zeron_visual::seed_for(&self.slot.id); + match Visual::load(&script, seed) { + Ok(visual) => { + self.visual = Some(visual.with_assets(zeron_visual::Assets::new(files))); + self.load = Load::Ready; + self.rerender(); + } + Err(error) => self.load = Load::Failed(format!("Visual failed: {error}")), + } + } + + // ---- tree, layout, display ------------------------------------------------- + + /// Re-run the view function after state changed. + fn rerender(&mut self) { + let Some(visual) = self.visual.as_mut() else { + return; + }; + match visual.render() { + Ok(tree) => { + self.tree = Some(tree); + self.generation += 1; + self.error = None; + } + Err(error) => self.error = Some(error.message), + } + } + + fn dispatch(&mut self, handler: HandlerId, arg: HandlerArg) -> bool { + let Some(visual) = self.visual.as_mut() else { + return false; + }; + match visual.call(handler, arg) { + Ok(()) => self.rerender(), + Err(error) => self.error = Some(error.message), + } + true + } + + fn status(&mut self, message: &str, color: Rgba, laid: &mut Laid, y: f32) -> f32 { + let mut ctx = TextCtx { + typo: &mut self.typo, + cache: &mut self.cache, + }; + let para = ctx.line( + message, + Face::new(zeron_visual::TextSize::Sm, zeron_visual::Weight::Regular), + color, + ); + laid.paint.para( + &mut ctx, + ¶, + 0.0, + y, + self.width, + zeron_visual::Align::Start, + false, + ) + } + + fn relayout(&mut self) { + if self.width <= 0.0 { + return; + } + let mut laid = match (&self.load, &self.tree) { + (Load::Ready, Some(tree)) => { + let mut ctx = TextCtx { + typo: &mut self.typo, + cache: &mut self.cache, + }; + layout::lay_out(&mut ctx, &self.palette, tree, self.width) + } + (Load::Failed(message), _) => { + let (message, color) = (message.clone(), self.palette.muted); + let mut laid = Laid::default(); + laid.height = self.status(&message, color, &mut laid, 0.0); + laid + } + _ => { + let color = self.palette.faint; + let mut laid = Laid::default(); + laid.height = self.status("Loading visual…", color, &mut laid, 0.0); + laid + } + }; + if let Some(error) = self.error.clone() { + let top = if laid.height > 0.0 { + laid.height + 8.0 + } else { + 0.0 + }; + let color = self.palette.danger; + laid.height = + top + self.status(&format!("Visual error: {error}"), color, &mut laid, top); + } + // Keep each surface's state (camera, buffers, touch) across renders. + let mut surfaces = HashMap::with_capacity(laid.surfaces.len()); + for spec in &laid.surfaces { + let state = match self.surfaces.remove(&spec.id) { + Some(mut state) => { + if state.spec != *spec { + state.spec = spec.clone(); + state.dirty = true; + } + state + } + None => SurfaceState::new(spec.clone()), + }; + surfaces.insert(spec.id, state); + } + // A new tree or palette changes what every surface draws. + for state in surfaces.values_mut() { + state.dirty = true; + } + self.surfaces = surfaces; + let height = laid.height.ceil(); + self.laid = laid; + self.display_dirty = true; + if self.slot.set(self.width, height) { + (self.changed)(self.row_key); + } + } + + fn display(&self) -> VisualDisplay { + VisualDisplay { + revision: self.revision, + width: self.width, + height: self.laid.height.ceil(), + dark: self.palette.dark, + paint: self.laid.paint.snapshot(), + surfaces: self + .laid + .surfaces + .iter() + .map(|spec| { + let reads = self.surfaces.get(&spec.id).is_some_and(|s| s.reads_pointer); + VisualSurface { + id: spec.id, + x: spec.rect[0], + y: spec.rect[1], + w: spec.rect[2], + h: spec.rect[3], + gesture: layout::gesture(&spec.node, reads), + } + }) + .collect(), + controls: self + .laid + .controls + .iter() + .map(|c| c.control.clone()) + .collect(), + styles: self.typo.table().to_vec(), + } + } + + /// Send what changed: the display, then due frames. + fn publish(&mut self) { + let Some(listener) = self.listener.clone() else { + return; + }; + // Frames first learn whether a canvas reads the pointer, which the + // display reports; a draw error is laid out under the visual. + let error = self.error.clone(); + let mut frames = self.frames(); + if self.error != error { + self.relayout(); + frames.extend(self.frames()); + } + if self.display_dirty { + self.display_dirty = false; + self.revision += 1; + listener.display(self.display()); + } + for frame in frames { + listener.frame(frame); + } + } + + fn time(&self) -> f64 { + self.started.elapsed().as_secs_f64() + } + + fn moving(&self, state: &SurfaceState) -> bool { + if self.reduce_motion { + return false; + } + let dragging = state.orbit.as_ref().is_some_and(|o| o.drag.is_some()); + match &state.spec.node { + SurfaceNode::Canvas(canvas) => canvas.animate, + SurfaceNode::Scene(scene) => scene.animate || (scene.spin != 0.0 && !dragging), + SurfaceNode::Shader(shader) => { + shader.animate || !shader.buffers.is_empty() || (shader.spin != 0.0 && !dragging) + } + SurfaceNode::Chart { .. } => false, + } + } + + /// How long to wait for commands before the next animation frame (or + /// load retry). + fn next_frame(&self) -> Option { + let retry = self + .retry + .map(|(at, _)| at.saturating_duration_since(Instant::now())); + let frame = self.listener.as_ref().and_then(|_| { + self.surfaces + .values() + .any(|s| self.moving(s)) + .then(|| FRAME.saturating_sub(self.last_frame.elapsed())) + }); + match (frame, retry) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } + } + + // ---- frames ------------------------------------------------------------------- + + fn frames(&mut self) -> Vec { + let ids: Vec = self.laid.surfaces.iter().map(|s| s.id).collect(); + let mut out = Vec::new(); + let mut rendered = false; + for id in ids { + let Some(state) = self.surfaces.get(&id) else { + continue; + }; + // Moving surfaces draw at the frame rate, however often commands + // wake the worker; changed ones draw now. + if !(state.dirty || self.moving(state) && self.last_frame.elapsed() >= FRAME) { + continue; + } + rendered = true; + if let Some(frame) = self.frame(id) { + out.push(frame); + } + } + if rendered { + self.last_frame = Instant::now(); + } + out + } + + fn frame(&mut self, id: u64) -> Option { + let mut state = self.surfaces.remove(&id)?; + state.dirty = false; + let frame = match state.spec.node.clone() { + SurfaceNode::Canvas(canvas) => self.canvas_frame(&mut state, &canvas), + SurfaceNode::Chart { chart, legend } => { + let mut ctx = TextCtx { + typo: &mut self.typo, + cache: &mut self.cache, + }; + let paint = chart::paint( + &mut ctx, + &self.palette, + &chart, + state.spec.rect[2], + legend, + state.hover, + ); + Some((None, paint)) + } + SurfaceNode::Scene(scene) => self.scene_frame(&mut state, &scene), + SurfaceNode::Shader(shader) => self.shader_frame(&mut state, &shader), + }; + self.surfaces.insert(id, state); + let (image, paint) = frame?; + Some(SurfaceFrame { + surface: id, + image, + paint: paint.finish(), + styles: self.typo.table().to_vec(), + }) + } + + fn canvas_frame( + &mut self, + state: &mut SurfaceState, + canvas: &zeron_visual::Canvas, + ) -> Option<(Option, Paint)> { + let (w, h) = state.spec.size(); + let pointer = state.pointer.map(|[x, y]| zeron_visual::Pointer { + x, + y, + down: state.down, + }); + let time = self.time(); + let drawn = self + .visual + .as_mut()? + .draw_canvas_traced(canvas.draw, w, h, time, pointer); + let drawing = match drawn { + Ok(drawing) => drawing, + Err(error) => { + self.error = Some(error.message); + return None; + } + }; + if drawing.reads_pointer != state.reads_pointer { + state.reads_pointer = drawing.reads_pointer; + self.display_dirty = true; + } + let mut ctx = TextCtx { + typo: &mut self.typo, + cache: &mut self.cache, + }; + let mut paint = Paint::default(); + for command in &drawing.commands { + match command { + DrawCmd::Stroke { + points, + closed, + color, + width, + dash, + } => paint.stroke( + points.iter().copied(), + *closed, + *width, + *dash, + self.palette.color(*color), + ), + DrawCmd::Fill { points, color } => { + paint.fill(points.iter().copied(), self.palette.color(*color)) + } + DrawCmd::Text(text) => { + let face = Face { + size: text.size, + weight: text.weight, + mono: text.mono, + italic: false, + }; + paint.label( + &mut ctx, + &text.text, + face, + self.palette.color(text.color), + [text.x, text.y], + text.align, + text.baseline, + ); + } + } + } + Some((None, paint)) + } + + fn scene_frame( + &mut self, + state: &mut SurfaceState, + node: &zeron_visual::SceneNode, + ) -> Option<(Option, Paint)> { + let (w, h) = state.spec.size(); + let animate = node.animate && !self.reduce_motion; + let fresh = state + .scene + .as_ref() + .is_some_and(|(generation, _)| *generation == self.generation && !animate); + if !fresh { + let time = self.time(); + match self.visual.as_mut()?.draw_scene(node.draw, time) { + Ok(scene) => state.scene = Some((self.generation, scene)), + Err(error) => { + self.error = Some(error.message); + return None; + } + } + } + let (_, scene) = state.scene.as_ref()?; + let spin = if self.reduce_motion { 0.0 } else { node.spin }; + let orbit = state.orbit.as_ref()?; + let camera = zeron_visual::Camera { + yaw: orbit.yaw(spin), + pitch: orbit.pitch, + zoom: node.zoom, + }; + let palette = self.palette; + let rgba = move |color| { + let c = palette.color(color); + [c.r, c.g, c.b, c.a] + }; + let pixel_scale = self.source.pixel_scale.min(MAX_GPU_SCALE); + let (image, hints, labels) = + match zeron_visual_gpu::render_scene(scene, camera, w, h, pixel_scale, &rgba) { + Some(frame) => { + let (hints, labels) = scene.annotations(camera, w, h); + ( + image_of(frame.width, frame.height, frame.bgra), + hints, + labels, + ) + } + None => { + // No GPU: the software renderer, rougher while it moves. + let quality = if self.moving(state) || orbit.drag.is_some() { + zeron_visual::Quality::Draft + } else { + zeron_visual::Quality::Final + }; + let raster = scene.render(camera, w, h, pixel_scale, quality, &rgba); + ( + image_of(raster.width, raster.height, raster.bgra), + raster.hints, + raster.labels, + ) + } + }; + state.hints = hints; + let mut ctx = TextCtx { + typo: &mut self.typo, + cache: &mut self.cache, + }; + let mut paint = Paint::default(); + for label in &labels { + let face = Face::new(label.size, zeron_visual::Weight::Regular); + paint.label( + &mut ctx, + &label.text, + face, + palette.color(label.color), + label.at, + zeron_visual::Align::Center, + zeron_visual::Align::End, + ); + } + if let Some(hint) = &state.hint { + let face = Face::new(zeron_visual::TextSize::Sm, zeron_visual::Weight::Medium); + paint.label( + &mut ctx, + hint, + face, + palette.text, + [0.0, 0.0], + zeron_visual::Align::Start, + zeron_visual::Align::Start, + ); + } + Some((Some(image), paint)) + } + + fn shader_frame( + &mut self, + state: &mut SurfaceState, + node: &zeron_visual::ShaderNode, + ) -> Option<(Option, Paint)> { + let (w, h) = state.spec.size(); + let scale = self.source.pixel_scale.min(MAX_GPU_SCALE); + let size = ( + (w * scale).round().max(1.0) as u32, + (h * scale).round().max(1.0) as u32, + ); + let animate = node.animate && !self.reduce_motion; + let spin = if self.reduce_motion { 0.0 } else { node.spin }; + let orbit = state.orbit.as_ref()?; + let yaw = orbit.yaw(spin); + // Shadertoy's mouse: pixels from the bottom-left; xy where the touch + // is, zw where it went down while it's down (negative after). + let device = |[x, y]: [f32; 2]| [x * scale, size.1 as f32 - y * scale]; + let mouse = match (state.pointer, state.press) { + (Some(now), Some(start)) if state.down => { + let ([x, y], [sx, sy]) = (device(now), device(start)); + [x, y, sx, sy] + } + (Some(now), _) => { + let [x, y] = device(now); + [x, y, -1.0, -1.0] + } + _ => [0.0, 0.0, -1.0, -1.0], + }; + let mut inputs = [[0.0f32; 4]; zeron_visual::shader::MAX_INPUTS / 4]; + for (i, (_, value)) in node.inputs.iter().enumerate() { + inputs[i / 4][i % 4] = *value; + } + let time = self.time(); + let shown_time = if animate { time } else { state.shader_time }; + let delta = (time - state.shader_time).max(0.0); + let uniforms = zeron_visual::shader::ShaderUniforms { + resolution: [size.0 as f32, size.1 as f32, scale, 0.0], + mouse, + camera: [yaw, orbit.pitch, node.zoom, 0.0], + time: [ + shown_time as f32, + delta as f32, + state.shader_frame as f32, + 0.0, + ], + inputs, + channels: Default::default(), + }; + let names: Vec = node.inputs.iter().map(|(n, _)| n.clone()).collect(); + let assets = self.visual.as_ref()?.assets().clone(); + let textures: Vec = node + .textures + .iter() + // A missing one keeps its slot (and samples as clear). + .map(|name| { + zeron_visual::TextureData::Encoded( + assets.get(name).unwrap_or_else(|| Arc::from([])), + ) + }) + .collect(); + let rendered = zeron_visual_gpu::render_shader( + &zeron_visual_gpu::ShaderSpec { + image: &node.code, + buffers: &node.buffers, + inputs: &names, + blend: node.blend, + bloom: node.bloom, + textures: &textures, + }, + &uniforms, + size.0, + size.1, + &mut state.shader, + ); + state.shader_frame += 1; + if animate || !node.buffers.is_empty() { + state.shader_time = time; + } + match rendered { + Ok(frame) => Some(( + Some(image_of(frame.width, frame.height, frame.bgra)), + Paint::default(), + )), + Err(error) => { + self.error = Some(error); + self.display_dirty = true; + None + } + } + } + + // ---- input ------------------------------------------------------------------ + + /// A touch on a surface; true when the tree must be laid out again. + fn pointer(&mut self, id: u64, phase: PointerPhase, at: [f32; 2]) -> bool { + let Some(mut state) = self.surfaces.remove(&id) else { + return false; + }; + state.dirty = true; + let mut call = None; + match phase { + PointerPhase::Down => { + state.pointer = Some(at); + state.down = true; + state.press = Some(at); + state.moved = false; + let spin = match &state.spec.node { + SurfaceNode::Scene(s) => s.spin, + SurfaceNode::Shader(s) => s.spin, + _ => 0.0, + }; + let orbits = matches!(&state.spec.node, SurfaceNode::Scene(s) if s.orbit) + || matches!(&state.spec.node, SurfaceNode::Shader(s) if s.orbit); + if let Some(orbit) = state.orbit.as_mut() + && orbits + { + orbit.yaw = orbit.yaw(if self.reduce_motion { 0.0 } else { spin }); + orbit.drag = Some((at, orbit.yaw, orbit.pitch)); + } + } + PointerPhase::Move => { + state.pointer = Some(at); + if let Some([sx, sy]) = state.press + && (at[0] - sx).hypot(at[1] - sy) > 8.0 + { + state.moved = true; + } + if let Some(orbit) = state.orbit.as_mut() + && let Some(([sx, sy], yaw, pitch)) = orbit.drag + { + orbit.yaw = yaw - (at[0] - sx) * 0.01; + orbit.pitch = (pitch + (at[1] - sy) * 0.01).clamp(-1.5, 1.5); + } + } + PointerPhase::Up | PointerPhase::Cancel => { + state.pointer = Some(at); + state.down = false; + if let Some(orbit) = state.orbit.as_mut() + && orbit.drag.take().is_some() + { + orbit.spin_epoch = Instant::now(); + } + let tap = phase == PointerPhase::Up && !state.moved; + match &state.spec.node { + SurfaceNode::Canvas(canvas) if tap => { + call = canvas + .on_click + .map(|h| (h, HandlerArg::Point(at[0], at[1]))); + } + SurfaceNode::Scene(_) if tap => { + // Labels a touch lands near, a fingertip wide. + state.hint = state + .hints + .iter() + .map(|([hx, hy], r, text)| ((hx - at[0]).hypot(hy - at[1]), *r, text)) + .filter(|(d, r, _)| *d <= r + 16.0) + .min_by(|a, b| a.0.total_cmp(&b.0)) + .map(|(_, _, text)| text.clone()); + } + _ => {} + } + } + } + if let SurfaceNode::Chart { chart, legend } = &state.spec.node { + state.hover = if state.down { + let mut ctx = TextCtx { + typo: &mut self.typo, + cache: &mut self.cache, + }; + chart::hover_at(&mut ctx, chart, state.spec.rect[2], *legend, at) + } else { + chart::Hover::default() + }; + } + self.surfaces.insert(id, state); + match call { + Some((handler, arg)) => self.dispatch(handler, arg), + None => false, + } + } + + fn control(&self, id: u64) -> Option { + self.laid + .controls + .iter() + .find(|c| c.control.id == id) + .cloned() + } + + fn activate(&mut self, id: u64) -> bool { + let Some(control) = self.control(id) else { + return false; + }; + match control.action { + Action::Call(handler, ActionArg::None) => self.dispatch(handler, HandlerArg::None), + Action::Call(handler, ActionArg::Bool(value)) => { + self.dispatch(handler, HandlerArg::Bool(value)) + } + Action::Call(handler, ActionArg::Text(value)) => { + self.dispatch(handler, HandlerArg::Text(value)) + } + Action::Call(_, ActionArg::Slider { .. }) | Action::Link => false, + } + } + + fn slide(&mut self, id: u64, fraction: f32) -> bool { + let Some(control) = self.control(id) else { + return false; + }; + let ( + Action::Call(handler, ActionArg::Slider { min, max, step }), + ControlKind::Slider { value: current, .. }, + ) = (control.action, control.control.kind) + else { + return false; + }; + let value = slider_value(min, max, step, fraction); + if (value - current).abs() <= f64::EPSILON { + return false; + } + self.dispatch(handler, HandlerArg::Number(value)) + } +} + +/// The value at `fraction` of a slider's track, on its steps. +fn slider_value(min: f64, max: f64, step: f64, fraction: f32) -> f64 { + let mut value = min + (max - min) * f64::from(fraction.clamp(0.0, 1.0)); + if step > 0.0 { + value = min + ((value - min) / step).round() * step; + } + value.clamp(min, max) +} + +/// Straight BGRA from the renderers → premultiplied for the platform. +fn image_of(width: u32, height: u32, mut bgra: Vec) -> FrameImage { + for px in bgra.chunks_exact_mut(4) { + let a = u16::from(px[3]); + if a < 255 { + for c in &mut px[..3] { + *c = ((u16::from(*c) * a + 127) / 255) as u8; + } + } + } + FrameImage { + width, + height, + bgra, + } +} + +/// Drive a future to completion on this (non-async) thread. +fn block_on(future: F) -> F::Output { + use std::task::{Context, Poll, Wake, Waker}; + struct Unpark(std::thread::Thread); + impl Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + } + let waker = Waker::from(Arc::new(Unpark(std::thread::current()))); + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(out) = future.as_mut().poll(&mut cx) { + return out; + } + std::thread::park(); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/mobile/src/visual/paint.rs b/crates/mobile/src/visual/paint.rs new file mode 100644 index 0000000000..21c21144d4 --- /dev/null +++ b/crates/mobile/src/visual/paint.rs @@ -0,0 +1,388 @@ +//! Vector paint for visuals: positioned text runs (measured here with the +//! bundled faces, drawn by the platform with the same bytes) and shapes. + +use zeron_text::{OverflowWrap, PrepareOptions, Prepared, Span, WhiteSpace, WidthCache}; +use zeron_visual::{Align, TextSize}; + +use super::{Rgba, VisualPaint, VisualRun, VisualShape}; +use crate::layout::style::{Family, Typography, Weight, baseline}; + +/// Text measurement for one visual (its own faces and width cache). +pub(crate) struct TextCtx<'a> { + pub typo: &'a mut Typography, + pub cache: &'a mut WidthCache, +} + +/// One styled stretch of a paragraph. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Face { + pub size: TextSize, + pub weight: zeron_visual::Weight, + pub mono: bool, + pub italic: bool, +} + +impl Face { + pub fn new(size: TextSize, weight: zeron_visual::Weight) -> Self { + Self { + size, + weight, + mono: false, + italic: false, + } + } +} + +fn weight(w: zeron_visual::Weight) -> Weight { + match w { + zeron_visual::Weight::Regular => Weight::Regular, + zeron_visual::Weight::Medium => Weight::Medium, + zeron_visual::Weight::Semibold => Weight::Semibold, + zeron_visual::Weight::Bold => Weight::Bold, + } +} + +/// A prepared paragraph: wraps at any width without re-measuring. +pub(crate) struct Para { + pub p: Prepared, + /// Line height and baseline within it. + pub lh: f32, + pub base: f32, + pub colors: Vec, +} + +impl TextCtx<'_> { + /// Prepare `parts` as one paragraph; the line height is the largest + /// face's. + pub fn para(&mut self, parts: &[(&str, Face, Rgba)]) -> Para { + let mut text = String::new(); + let mut spans = Vec::new(); + let mut colors = Vec::new(); + let mut lh: f32 = 0.0; + let mut base: f32 = 0.0; + for (s, face, color) in parts { + let (size, line) = face.size.metrics(); + let family = if face.mono { + Family::Mono + } else { + Family::Sans + }; + let style = self + .typo + .style(family, weight(face.weight), face.italic, size); + let line = self.typo.px(line); + if line >= lh { + lh = line; + base = baseline(line, style); + } + if s.is_empty() { + continue; + } + let start = text.len(); + text.push_str(s); + spans.push(Span::new(start..text.len(), style.id)); + colors.push(*color); + } + let p = zeron_text::prepare( + &self.typo.book, + self.cache, + &text, + &spans, + &PrepareOptions { + white_space: WhiteSpace::PreWrap, + overflow_wrap: OverflowWrap::Anywhere, + ..Default::default() + }, + ); + Para { + p, + lh, + base, + colors, + } + } + + /// One line of plain text. + pub fn line(&mut self, text: &str, face: Face, color: Rgba) -> Para { + self.para(&[(text, face, color)]) + } + + pub fn line_height(&mut self, size: TextSize) -> f32 { + self.typo.px(size.metrics().1) + } +} + +impl Para { + /// Its width unwrapped. + pub fn natural(&self) -> f32 { + self.p.max_content_width() + } +} + +/// Accumulates a layer of runs and shapes. +#[derive(Default)] +pub(crate) struct Paint { + pub text: String, + utf16: u32, + pub runs: Vec, + pub shapes: Vec, +} + +/// Where a paint stood, so what was added since can be moved. +#[derive(Clone, Copy)] +pub(crate) struct Mark { + runs: usize, + shapes: usize, +} + +impl Paint { + pub fn mark(&self) -> Mark { + Mark { + runs: self.runs.len(), + shapes: self.shapes.len(), + } + } + + /// Move what was added between two marks. + pub fn shift_between(&mut self, from: Mark, to: Mark, dx: f32, dy: f32) { + if dx == 0.0 && dy == 0.0 { + return; + } + for run in &mut self.runs[from.runs..to.runs] { + run.x += dx; + run.baseline += dy; + } + for shape in &mut self.shapes[from.shapes..to.shapes] { + match shape { + VisualShape::Rect { x, y, .. } => { + *x += dx; + *y += dy; + } + VisualShape::Path { points, .. } => { + for xy in points.chunks_exact_mut(2) { + xy[0] += dx; + xy[1] += dy; + } + } + } + } + } + + pub fn rect(&mut self, x: f32, y: f32, w: f32, h: f32, radius: f32, color: Rgba) { + if w <= 0.0 || h <= 0.0 || color.a <= 0.0 { + return; + } + self.shapes.push(VisualShape::Rect { + x, + y, + w, + h, + radius, + color, + }); + } + + pub fn fill(&mut self, points: impl IntoIterator, color: Rgba) { + let points: Vec = points.into_iter().flatten().collect(); + if points.len() < 6 || color.a <= 0.0 { + return; + } + self.shapes.push(VisualShape::Path { + points, + closed: true, + fill: Some(color), + stroke: None, + line_width: 0.0, + dash: None, + }); + } + + pub fn stroke( + &mut self, + points: impl IntoIterator, + closed: bool, + width: f32, + dash: Option, + color: Rgba, + ) { + let points: Vec = points.into_iter().flatten().collect(); + if points.len() < 4 || color.a <= 0.0 { + return; + } + self.shapes.push(VisualShape::Path { + points, + closed, + fill: None, + stroke: Some(color), + line_width: width, + dash, + }); + } + + /// A filled dot (chart points). + pub fn dot(&mut self, [cx, cy]: [f32; 2], radius: f32, color: Rgba) { + self.rect( + cx - radius, + cy - radius, + radius * 2.0, + radius * 2.0, + radius, + color, + ); + } + + /// Wrap `para` at `width` from (x, y); returns its height. `truncate` + /// keeps the first line, ending in an ellipsis when more follows. + #[allow(clippy::too_many_arguments)] + pub fn para( + &mut self, + ctx: &mut TextCtx, + para: &Para, + x: f32, + y: f32, + width: f32, + align: Align, + truncate: bool, + ) -> f32 { + let width = width.max(1.0); + if truncate { + let lines = para.p.lines(width); + if lines.len() <= 1 { + self.lines(para, &lines, x, y, width, align); + return para.lh; + } + // Room for the ellipsis in the last span's face. + let last_style = para.p.spans().last().map(|s| s.style); + let ellipsis_width = last_style.map_or(0.0, |style| { + let p = zeron_text::prepare( + &ctx.typo.book, + ctx.cache, + "…", + &[Span::new(0.."…".len(), style)], + &PrepareOptions::default(), + ); + p.max_content_width() + }); + let lines = para.p.lines((width - ellipsis_width).max(1.0)); + let first = &lines[..1]; + let end = self.lines(para, first, x, y, width - ellipsis_width, Align::Start); + if let Some(style) = last_style { + let start = self.push_text("…"); + let color = para.colors.last().copied().unwrap_or_default(); + self.runs.push(VisualRun { + start, + len: 1, + x: end, + baseline: y + para.base, + width: ellipsis_width, + style: style.0, + color, + }); + } + return para.lh; + } + let lines = para.p.lines(width); + self.lines(para, &lines, x, y, width, align); + lines.len().max(1) as f32 * para.lh + } + + /// Draw laid-out lines; returns where the last one ended. + fn lines( + &mut self, + para: &Para, + lines: &[zeron_text::Line], + x: f32, + y: f32, + width: f32, + align: Align, + ) -> f32 { + if lines.is_empty() { + return x; + } + let base16 = self.push_text(para.p.text()); + let spans = para.p.spans(); + let mut end = x; + for (i, line) in lines.iter().enumerate() { + let offset = match align { + Align::Start => 0.0, + Align::Center => ((width - line.width) / 2.0).max(0.0), + Align::End => (width - line.width).max(0.0), + }; + let baseline = y + i as f32 * para.lh + para.base; + for f in &line.fragments { + if f.utf16.is_empty() { + continue; + } + self.runs.push(VisualRun { + start: base16 + f.utf16.start as u32, + len: (f.utf16.end - f.utf16.start) as u32, + x: x + offset + f.x, + baseline, + width: f.width, + style: spans[f.span].style.0, + color: para.colors.get(f.span).copied().unwrap_or_default(), + }); + } + end = x + offset + line.width; + } + end + } + + /// One line anchored at `at`: `align` places it horizontally around x, + /// `vertical` places its line box below (Start), around (Center) or + /// above (End) y. Returns its width. + #[allow(clippy::too_many_arguments)] + pub fn label( + &mut self, + ctx: &mut TextCtx, + text: &str, + face: Face, + color: Rgba, + [x, y]: [f32; 2], + align: Align, + vertical: Align, + ) -> f32 { + if text.is_empty() { + return 0.0; + } + let para = ctx.line(text, face, color); + let width = para.natural(); + let left = match align { + Align::Start => x, + Align::Center => x - width / 2.0, + Align::End => x - width, + }; + let top = match vertical { + Align::Start => y, + Align::Center => y - para.lh / 2.0, + Align::End => y - para.lh, + }; + let lines = para.p.lines(1.0e7); + self.lines(¶, &lines, left, top, width, Align::Start); + width + } + + fn push_text(&mut self, s: &str) -> u32 { + let start = self.utf16; + self.text.push_str(s); + self.utf16 += s.encode_utf16().count() as u32; + start + } + + /// A copy for the platform (the layout keeps its own). + pub fn snapshot(&self) -> VisualPaint { + VisualPaint { + text: self.text.clone(), + runs: self.runs.clone(), + shapes: self.shapes.clone(), + } + } + + pub fn finish(self) -> VisualPaint { + VisualPaint { + text: self.text, + runs: self.runs, + shapes: self.shapes, + } + } +} diff --git a/crates/mobile/src/visual/palette.rs b/crates/mobile/src/visual/palette.rs new file mode 100644 index 0000000000..5efe2aa41f --- /dev/null +++ b/crates/mobile/src/visual/palette.rs @@ -0,0 +1,198 @@ +//! The colors visuals paint with: the app palette for one appearance +//! (`apps/ios/Zeron/Design/Palette.swift`, the desktop's Zeron theme), and +//! the categorical chart series derived from its accent as the desktop does +//! (`crates/ui/src/visuals.rs`, `chart_series`). +//! +//! Visuals resolve colors in Rust, unlike transcript rows: scenes and +//! shaders bake them into GPU frames, so the painter only ever sees RGBA. + +use zeron_visual::{Color, Token}; + +use super::Rgba; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Palette { + pub dark: bool, + pub text: Rgba, + pub muted: Rgba, + pub faint: Rgba, + pub accent: Rgba, + pub on_accent: Rgba, + pub success: Rgba, + pub warning: Rgba, + pub danger: Rgba, + pub border: Rgba, + /// A stronger rule (a line chart's hover cursor). + pub border_strong: Rgba, + /// Translucent fill for pressed rows, secondary buttons and tracks. + pub wash: Rgba, + pub chart: [Rgba; 6], +} + +pub(crate) fn hex(v: u32) -> Rgba { + Rgba { + r: ((v >> 16) & 0xff) as f32 / 255.0, + g: ((v >> 8) & 0xff) as f32 / 255.0, + b: (v & 0xff) as f32 / 255.0, + a: 1.0, + } +} + +impl Rgba { + pub(crate) fn alpha(self, a: f32) -> Rgba { + Rgba { + a: self.a * a, + ..self + } + } +} + +impl Palette { + pub fn new(dark: bool) -> Self { + let pick = |light: u32, night: u32| hex(if dark { night } else { light }); + let text = pick(0x27272C, 0xE8E8EA); + let accent = pick(0x5B43E8, 0x8B7CF6); + Self { + dark, + text, + muted: pick(0x62626A, 0xA9A9AE), + faint: pick(0x97979F, 0x6B6B72), + accent, + on_accent: hex(0xFFFFFF), + success: pick(0x15803D, 0x34D399), + warning: pick(0xA16207, 0xFACC15), + danger: pick(0xDC2626, 0xF87171), + border: pick(0xE2E2E6, 0x232327), + border_strong: text.alpha(0.28), + wash: text.alpha(0.075), + chart: chart_series(accent, dark), + } + } + + pub fn token(&self, token: Token) -> Rgba { + match token { + Token::Text => self.text, + Token::Muted => self.muted, + Token::Faint => self.faint, + Token::Accent => self.accent, + Token::OnAccent => self.on_accent, + Token::Success => self.success, + Token::Warning => self.warning, + Token::Danger => self.danger, + Token::Border => self.border, + Token::Chart(i) => self.chart[usize::from(i) % 6], + Token::Rgb(rgb) => hex(rgb), + } + } + + pub fn color(&self, color: Color) -> Rgba { + let mut out = self.token(color.token); + if let Some((other, t)) = color.mix { + // In RGB, so a ramp between two hues doesn't swing through a third. + let b = self.token(other); + let lerp = |x: f32, y: f32| x + (y - x) * t; + out = Rgba { + r: lerp(out.r, b.r), + g: lerp(out.g, b.g), + b: lerp(out.b, b.b), + a: lerp(out.a, b.a), + }; + } + out.a *= color.alpha; + out + } +} + +/// Categorical chart colors: the accent, then the five fixed hues farthest +/// from it, so no two series read alike. Deeper on light, brighter on dark. +fn chart_series(accent: Rgba, dark: bool) -> [Rgba; 6] { + // (hue in turns, light tone, dark tone): teal, amber, rose, violet, lime, sky. + const FIXED: [(f32, u32, u32); 6] = [ + (0.486, 0x0d9488, 0x2dd4bf), + (0.106, 0xd97706, 0xfbbf24), + (0.964, 0xe11d48, 0xfb7185), + (0.728, 0x7c3aed, 0xa78bfa), + (0.236, 0x65a30d, 0xa3e635), + (0.553, 0x0284c7, 0x38bdf8), + ]; + let (hue, saturation) = hue_saturation(accent); + let distance = |h: f32| { + let d = (h - hue).rem_euclid(1.0); + d.min(1.0 - d) + }; + // A grey accent has no hue to collide with; drop the last fixed color. + let closest = if saturation < 0.15 { + FIXED.len() - 1 + } else { + (0..FIXED.len()) + .min_by(|a, b| distance(FIXED[*a].0).total_cmp(&distance(FIXED[*b].0))) + .unwrap_or(0) + }; + let mut out = [accent; 6]; + for (slot, (_, (_, light, night))) in FIXED + .iter() + .enumerate() + .filter(|(ix, _)| *ix != closest) + .enumerate() + { + out[slot + 1] = hex(if dark { *night } else { *light }); + } + out +} + +/// HSL hue (in turns) and saturation. +fn hue_saturation(c: Rgba) -> (f32, f32) { + let max = c.r.max(c.g).max(c.b); + let min = c.r.min(c.g).min(c.b); + let delta = max - min; + if delta <= f32::EPSILON { + return (0.0, 0.0); + } + let lightness = (max + min) / 2.0; + let saturation = delta / (1.0 - (2.0 * lightness - 1.0).abs()).max(f32::EPSILON); + let hue = if max == c.r { + ((c.g - c.b) / delta).rem_euclid(6.0) + } else if max == c.g { + (c.b - c.r) / delta + 2.0 + } else { + (c.r - c.g) / delta + 4.0 + } / 6.0; + (hue, saturation) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chart_series_are_six_distinct_colors_led_by_the_accent() { + for dark in [false, true] { + let palette = Palette::new(dark); + assert_eq!(palette.chart[0], palette.accent); + for (i, a) in palette.chart.iter().enumerate() { + for b in &palette.chart[i + 1..] { + assert_ne!(a, b); + } + } + // The violet accent drops the fixed violet. + assert!( + !palette + .chart + .contains(&hex(if dark { 0xa78bfa } else { 0x7c3aed })) + ); + } + } + + #[test] + fn colors_mix_in_rgb_and_carry_alpha() { + let palette = Palette::new(false); + let half = palette.color(Color::token(Token::Accent).with_alpha(0.5)); + assert!((half.a - 0.5).abs() < 1e-6); + let mixed = palette.color(Color { + token: Token::Rgb(0x000000), + mix: Some((Token::Rgb(0xffffff), 0.25)), + alpha: 1.0, + }); + assert!((mixed.r - 0.25).abs() < 1e-6 && (mixed.g - 0.25).abs() < 1e-6); + } +} diff --git a/crates/mobile/src/visual/tests.rs b/crates/mobile/src/visual/tests.rs new file mode 100644 index 0000000000..4d03c31d4a --- /dev/null +++ b/crates/mobile/src/visual/tests.rs @@ -0,0 +1,507 @@ +use std::collections::HashMap; +use std::sync::mpsc::{self, Receiver}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use zeron_text::WidthCache; + +use super::layout::{self, SurfaceNode}; +use super::paint::TextCtx; +use super::palette::Palette; +use super::*; + +enum Event { + Display(VisualDisplay), + Frame(SurfaceFrame), +} + +struct Collect(Mutex>); + +impl VisualListener for Collect { + fn display(&self, display: VisualDisplay) { + let _ = self.0.lock().unwrap().send(Event::Display(display)); + } + fn frame(&self, frame: SurfaceFrame) { + let _ = self.0.lock().unwrap().send(Event::Frame(frame)); + } +} + +const PATH: &str = "uploads/v_1-visual.luau"; + +struct Open { + _hub: Arc, + session: Arc, + slot: Arc, + events: Receiver, + remeasured: Receiver, +} + +fn open(script: &str, width: f32) -> Open { + let (changed_tx, remeasured) = mpsc::channel(); + let changed_tx = Mutex::new(changed_tx); + let hub = Arc::new(VisualHub::new( + crate::layout::tests::text_system(), + Arc::new(move |key| { + let _ = changed_tx.lock().unwrap().send(key); + }), + )); + let files: HashMap> = [(PATH.to_owned(), script.as_bytes().to_vec())].into(); + hub.set_fetch(Arc::new(move |path: &str| files.get(path).cloned()), 2.0); + let slot = VisualSlot::new("v_1", PATH, "Test"); + hub.ensure(&slot, 42, width, 1.0); + let session = hub.session("v_1").expect("spawned"); + let (tx, events) = mpsc::channel(); + session.attach(Arc::new(Collect(Mutex::new(tx))), false, false); + Open { + _hub: hub, + session, + slot, + events, + remeasured, + } +} + +impl Open { + /// The first display (and the frames before it) satisfying `want`. + fn display_where( + &self, + want: impl Fn(&VisualDisplay) -> bool, + ) -> (VisualDisplay, Vec) { + let deadline = Instant::now() + Duration::from_secs(10); + let mut frames = Vec::new(); + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match self.events.recv_timeout(left).expect("a display in time") { + Event::Display(d) if want(&d) => { + // Frames follow their display (animated ones keep coming). + let until = Instant::now() + Duration::from_millis(300); + while let Ok(event) = self + .events + .recv_timeout(until.saturating_duration_since(Instant::now())) + { + if let Event::Frame(f) = event { + frames.push(f); + } + } + return (d, frames); + } + Event::Display(_) => {} + Event::Frame(f) => frames.push(f), + } + } + } + + fn ready(&self) -> (VisualDisplay, Vec) { + self.display_where(|d| !d.paint.text.contains("Loading visual")) + } + + fn frame_where(&self, want: impl Fn(&SurfaceFrame) -> bool) -> SurfaceFrame { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let left = deadline.saturating_duration_since(Instant::now()); + if let Event::Frame(f) = self.events.recv_timeout(left).expect("a frame in time") + && want(&f) + { + return f; + } + } + } +} + +fn text_of(d: &VisualDisplay) -> String { + let units: Vec = d.paint.text.encode_utf16().collect(); + d.paint + .runs + .iter() + .map(|r| String::from_utf16_lossy(&units[r.start as usize..(r.start + r.len) as usize])) + .collect::>() + .join("|") +} + +fn lay(script: &str, width: f32) -> layout::Laid { + let mut visual = Visual::load(script, 1).unwrap(); + let tree = visual.render().unwrap(); + let text = crate::layout::tests::text_system(); + let mut typo = crate::layout::typography(&text); + let mut cache = WidthCache::new(); + let mut ctx = TextCtx { + typo: &mut typo, + cache: &mut cache, + }; + layout::lay_out(&mut ctx, &Palette::new(true), &tree, width) +} + +#[test] +fn rows_share_width_and_columns_stack_with_gaps() { + let laid = lay( + r#"return ui.column { gap = 10, + ui.row { gap = 8, + ui.stat { value = "182 ms", label = "p95" }, + ui.canvas { height = 60, draw = function(g) end }, + }, + ui.chart { type = "bar", height = 120, x = { "a", "b" }, series = { { name = "n", values = { 1, 2 } } } }, + }"#, + 300.0, + ); + assert_eq!(laid.surfaces.len(), 2); + let canvas = &laid.surfaces[0]; + let chart = &laid.surfaces[1]; + assert!(matches!(canvas.node, SurfaceNode::Canvas(_))); + // The canvas fills what the stat leaves of the row. + let [cx, cy, cw, ch] = canvas.rect; + assert!(cx > 40.0 && cy == 0.0 && ch == 60.0, "{:?}", canvas.rect); + assert!((cx + cw - 300.0).abs() < 0.5, "{:?}", canvas.rect); + // The chart sits a gap below the row, full width, legend band on top. + let [x, y, w, h] = chart.rect; + assert_eq!((x, w), (0.0, 300.0)); + assert!((y - 70.0).abs() < 0.5, "{y}"); + assert!(h > 120.0); + assert!((laid.height - (y + h)).abs() < 0.5); +} + +#[test] +fn rows_center_shorter_children_and_justify_between() { + let laid = lay( + r#"return ui.row { align = "center", justify = "between", + ui.button { label = "A", on_click = function() end }, + ui.canvas { height = 100, width = 50, draw = function(g) end }, + }"#, + 300.0, + ); + let button = &laid.controls[0].control; + assert_eq!(button.x, 0.0); + // Centered on the 100-tall canvas. + assert!( + (button.y - (100.0 - layout::metrics::BUTTON_H) / 2.0).abs() < 0.5, + "{}", + button.y + ); + assert!( + (laid.surfaces[0].rect[0] - 250.0).abs() < 0.5, + "{:?}", + laid.surfaces[0].rect + ); +} + +#[test] +fn wrapped_rows_and_grids_flow_onto_new_lines() { + let laid = lay( + r#"return ui.column { + ui.row { wrap = true, gap = 4, + ui.button { label = "First button", on_click = function() end }, + ui.button { label = "Second button", on_click = function() end }, + ui.button { label = "Third button", on_click = function() end }, + }, + ui.grid { columns = 2, gap = 6, + ui.canvas { height = 40, draw = function(g) end }, + ui.canvas { height = 40, draw = function(g) end }, + ui.canvas { height = 40, draw = function(g) end }, + }, + }"#, + 220.0, + ); + let ys: Vec = laid.controls.iter().map(|c| c.control.y).collect(); + assert!(ys[2] > ys[0], "{ys:?}"); + let cells: Vec<[f32; 4]> = laid.surfaces.iter().map(|s| s.rect).collect(); + assert_eq!(cells[0][1], cells[1][1]); + assert!((cells[0][2] - 107.0).abs() < 0.5, "{cells:?}"); + assert!((cells[2][1] - cells[0][1] - 46.0).abs() < 0.5, "{cells:?}"); +} + +#[test] +fn tables_lay_cells_in_columns_under_a_header() { + let laid = lay( + r#"return ui.table { columns = { "Region", { label = "p95", align = "end" } }, rows = { { "us-east", 182 }, { "eu-west", 140 } } }"#, + 300.0, + ); + let units: Vec = laid.paint.text.encode_utf16().collect(); + let run = |s: &str| { + laid.paint + .runs + .iter() + .find(|r| { + String::from_utf16_lossy(&units[r.start as usize..(r.start + r.len) as usize]) == s + }) + .unwrap_or_else(|| panic!("{s}")) + .clone() + }; + let (region, p95, east, value) = (run("Region"), run("p95"), run("us-east"), run("182")); + assert_eq!(region.x, layout::metrics::TABLE_CELL_PAD); + assert!(east.baseline > region.baseline); + // The second column ends its cells at its right padding. + let right = 300.0 - layout::metrics::TABLE_CELL_PAD; + assert!((value.x + value.width - right).abs() < 0.5, "{value:?}"); + assert!((p95.x + p95.width - right).abs() < 0.5, "{p95:?}"); +} + +#[test] +fn a_visual_loads_measures_its_row_and_reacts_to_controls() { + let open = open( + r#" + local s = ui.state { n = 0, on = false, v = 2 } + return function() + return ui.column { gap = 8, + ui.text { "Count " .. s.n }, + s.on and ui.text "Shown" or ui.space(0), + ui.button { label = "Add", on_click = function() s.n = s.n + 1 end }, + ui.toggle { label = "Show", value = s.on, on_change = function(v) s.on = v end }, + ui.slider { label = "Value", value = s.v, min = 0, max = 10, step = 1, on_change = function(v) s.v = v end }, + } + end + "#, + 320.0, + ); + let (display, _) = open.ready(); + assert!( + text_of(&display).contains("Count 0"), + "{}", + text_of(&display) + ); + // The transcript heard the visual's height. + assert_eq!( + open.remeasured + .recv_timeout(Duration::from_secs(5)) + .unwrap(), + 42 + ); + assert_eq!(open.slot.height(), Some(display.height)); + + let id = |kind: fn(&ControlKind) -> bool, d: &VisualDisplay| { + d.controls.iter().find(|c| kind(&c.kind)).unwrap().id + }; + open.session + .activate(id(|k| matches!(k, ControlKind::Button { .. }), &display)); + let (display, _) = open.display_where(|d| text_of(d).contains("Count 1")); + + open.session + .activate(id(|k| matches!(k, ControlKind::Toggle { .. }), &display)); + let (shown, _) = open.display_where(|d| text_of(d).contains("Shown")); + assert!(shown.height > display.height, "a new line makes it taller"); + assert!( + shown + .controls + .iter() + .any(|c| c.kind == ControlKind::Toggle { on: true }) + ); + + open.session.slide( + id(|k| matches!(k, ControlKind::Slider { .. }), &shown), + 0.74, + ); + open.display_where(|d| { + d.controls + .iter() + .any(|c| matches!(&c.kind, ControlKind::Slider { value, .. } if *value == 7.0)) + }); +} + +#[test] +fn commands_in_one_batch_each_see_the_tree_the_last_one_left() { + let open = open( + r#" + local s = ui.state { on = false } + return function() + return ui.toggle { label = "Flag", value = s.on, on_change = function(v) s.on = v end } + end + "#, + 200.0, + ); + let (display, _) = open.ready(); + let toggle = display.controls[0].id; + // A quick double tap: on, then off again (not on twice). + open.session.activate(toggle); + open.session.activate(toggle); + let deadline = Instant::now() + Duration::from_secs(5); + let mut last = None; + while Instant::now() < deadline { + match open.events.recv_timeout(Duration::from_millis(400)) { + Ok(Event::Display(d)) => last = Some(d.controls[0].kind.clone()), + Ok(Event::Frame(_)) => {} + Err(_) if last.is_some() => break, + Err(_) => {} + } + } + assert_eq!(last, Some(ControlKind::Toggle { on: false })); +} + +#[test] +fn a_tappable_stack_sits_under_the_controls_inside_it() { + let laid = lay( + r#"return ui.row { on_click = function() end, + ui.text "Row", + ui.button { label = "Go", on_click = function() end }, + }"#, + 300.0, + ); + let kinds: Vec<&ControlKind> = laid.controls.iter().map(|c| &c.control.kind).collect(); + assert_eq!( + kinds, + [&ControlKind::Tap, &ControlKind::Button { primary: false }] + ); +} + +#[test] +fn a_script_the_host_couldnt_send_loads_on_a_retry() { + let tries = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let hub = Arc::new(VisualHub::new( + crate::layout::tests::text_system(), + Arc::new(|_| {}), + )); + let counter = tries.clone(); + hub.set_fetch( + Arc::new(move |path: &str| { + // Unreachable at first, then there. + let first = counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0; + (!first && path == PATH).then(|| br#"return ui.text "Back""#.to_vec()) + }), + 2.0, + ); + let slot = VisualSlot::new("v_1", PATH, "Test"); + hub.ensure(&slot, 1, 300.0, 1.0); + let (tx, events) = mpsc::channel(); + hub.session("v_1") + .unwrap() + .attach(Arc::new(Collect(Mutex::new(tx))), false, false); + let deadline = Instant::now() + Duration::from_secs(8); + loop { + if let Event::Display(d) = events + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("a display") + && text_of(&d) == "Back" + { + break; + } + } +} + +#[test] +fn canvases_that_read_the_pointer_take_drags_and_get_touches() { + let open = open( + r#" + return ui.canvas { height = 80, draw = function(g, w, h) + local p = g.pointer + if p then g:text(p.x, p.y, string.format("%d,%d", p.x, p.y)) end + end } + "#, + 200.0, + ); + let (display, _) = open.display_where(|d| { + d.surfaces + .first() + .is_some_and(|s| s.gesture == SurfaceGesture::Drag) + }); + let surface = display.surfaces[0].id; + open.session + .pointer(surface, PointerPhase::Down, 30.0, 40.0); + open.frame_where(|f| f.paint.text.contains("30,40")); +} + +#[test] +fn charts_read_out_the_category_under_a_finger() { + let open = open( + r#"return ui.chart { type = "bar", height = 120, x = { "Mon", "Tue" }, + series = { { name = "a", values = { 3, 5 } }, { name = "b", values = { 1, 2 } } } }"#, + 300.0, + ); + let (display, frames) = open.ready(); + let chart = &display.surfaces[0]; + assert_eq!(chart.gesture, SurfaceGesture::HorizontalDrag); + assert!( + frames + .iter() + .any(|f| f.surface == chart.id && !f.paint.shapes.is_empty()) + ); + // Over the right half of the plot: Tuesday, with both series' values. + open.session + .pointer(chart.id, PointerPhase::Down, chart.w * 0.8, chart.h * 0.6); + let readout = open.frame_where(|f| f.paint.text.starts_with("Tue")); + assert!( + readout.paint.text.contains('5') && readout.paint.text.contains('2'), + "{}", + readout.paint.text + ); + // Lifting the finger clears it. + open.session + .pointer(chart.id, PointerPhase::Up, chart.w * 0.8, chart.h * 0.6); + open.frame_where(|f| !f.paint.text.starts_with("Tue")); +} + +#[test] +fn scenes_and_shaders_render_premultiplied_frames_on_the_gpu() { + if !zeron_visual_gpu::available() { + return; + } + let open = open( + r#"return ui.column { gap = 8, + ui.scene { height = 120, draw = function(s) s:box { at = { 0, 0.5, 0 }, size = 1, color = "accent" } end }, + ui.shader { height = 60, code = [[ + void mainImage(out vec4 fragColor, in vec2 fragCoord) { fragColor = vec4(1.0, 0.5, 0.0, 0.5); } + ]], blend = "alpha" }, + }"#, + 200.0, + ); + let (display, frames) = open.ready(); + assert_eq!(display.surfaces.len(), 2); + for surface in &display.surfaces { + let frame = frames + .iter() + .rev() + .find(|f| f.surface == surface.id) + .expect("a frame per surface"); + let image = frame.image.as_ref().expect("an image"); + // Rendered at the pixel scale given (2×). + assert_eq!( + (image.width, image.height), + ((surface.w * 2.0) as u32, (surface.h * 2.0) as u32) + ); + // Premultiplied: no channel above its alpha. + assert!( + image + .bgra + .chunks_exact(4) + .all(|p| p[0] <= p[3] && p[1] <= p[3] && p[2] <= p[3]) + ); + assert!( + image.bgra.chunks_exact(4).any(|p| p[3] > 0), + "something drew" + ); + } +} + +#[test] +fn a_missing_script_says_so() { + let hub = Arc::new(VisualHub::new( + crate::layout::tests::text_system(), + Arc::new(|_| {}), + )); + hub.set_fetch(Arc::new(|_: &str| None), 2.0); + let slot = VisualSlot::new("gone", "uploads/gone-visual.luau", "Gone"); + hub.ensure(&slot, 1, 300.0, 1.0); + let (tx, events) = mpsc::channel(); + hub.session("gone") + .unwrap() + .attach(Arc::new(Collect(Mutex::new(tx))), false, false); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Event::Display(d) = events + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("a display") + && text_of(&d) == "Visual unavailable" + { + break; + } + } +} + +#[test] +fn sliders_land_on_steps_within_range() { + assert_eq!(slider_value(0.0, 10.0, 1.0, 0.74), 7.0); + assert_eq!(slider_value(0.0, 1.0, 0.25, 0.6), 0.5); + assert_eq!(slider_value(5.0, 6.0, 0.0, 2.0), 6.0); +} + +#[test] +fn frames_are_premultiplied_for_core_graphics() { + let image = image_of(2, 1, vec![200, 100, 50, 128, 10, 20, 30, 255]); + assert_eq!(image.bgra, vec![100, 50, 25, 128, 10, 20, 30, 255]); +} diff --git a/crates/proto/src/agent.rs b/crates/proto/src/agent.rs index d74c330528..6ce7b0e855 100644 --- a/crates/proto/src/agent.rs +++ b/crates/proto/src/agent.rs @@ -419,6 +419,27 @@ pub const SUBAGENT_INPUT_KEEP: [&str; 5] = [ /// script Zeron runs in a sandbox and renders natively. pub const VISUAL_MAX_BYTES: usize = 256 * 1024; pub const VISUAL_MAX_TITLE_CHARS: usize = 200; +/// Files a visual may ship with (models, textures, environment maps). +pub const VISUAL_MAX_ASSETS: usize = 16; +pub const VISUAL_MAX_ASSET_BYTES: usize = 8 * 1024 * 1024; +pub const VISUAL_MAX_TOTAL_ASSET_BYTES: usize = 10 * 1024 * 1024; + +// The MCP server sends a visual to the engine in one WebSocket message, its +// assets base64-encoded; that message must fit the engine's 16 MiB frame +// (tungstenite's default), with room for the script escaped as JSON. +const _: () = assert!( + VISUAL_MAX_TOTAL_ASSET_BYTES.div_ceil(3) * 4 + VISUAL_MAX_BYTES * 6 + 64 * 1024 < 16 * 1024 * 1024 +); + +/// Whether `name` can name a visual's asset: a plain file name. +pub fn valid_visual_asset_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 64 + && !name.starts_with('.') + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) +} /// Where one checklist item stands. Claude's TodoWrite, OpenCode and ACP plans /// distinguish the item being worked on from those still waiting; Codex and diff --git a/crates/text/src/font.rs b/crates/text/src/font.rs index 1eebea73b6..cd8a3f8839 100644 --- a/crates/text/src/font.rs +++ b/crates/text/src/font.rs @@ -208,7 +208,15 @@ impl FontBook { } // Validate before leaking so a bad file doesn't leak. ttf_parser::Face::parse(&data, 0).map_err(|_| FontError::Parse)?; - let data: &'static [u8] = Box::leak(data.into_boxed_slice()); + self.add_static_face(Box::leak(data.into_boxed_slice())) + } + + /// Registers a font file already in memory for the process, so many books + /// can share one copy. + pub fn add_static_face(&mut self, data: &'static [u8]) -> Result { + if self.faces.len() >= u16::MAX as usize { + return Err(FontError::TooManyFaces); + } let hb = rustybuzz::Face::from_slice(data, 0).ok_or(FontError::Parse)?; let upem = hb.units_per_em() as f32; let mut ascii = 0u128; diff --git a/crates/ui/src/visuals.rs b/crates/ui/src/visuals.rs index a61a4f2763..a07ce718ba 100644 --- a/crates/ui/src/visuals.rs +++ b/crates/ui/src/visuals.rs @@ -67,6 +67,8 @@ struct Local { /// The pointer over a canvas, chart, or scene, in window coordinates. pointers: HashMap>, pressed: Option, + /// Where the pointer went down on a shader, while it's held (iMouse.zw). + presses: HashMap>, /// Where each chart's plot was painted, for hover hit-testing. charts: HashMap, cameras: HashMap, @@ -158,21 +160,56 @@ impl VisualFrame { target_device_id: Option, cx: &mut Context, ) -> Self { + // Its drawings leave every window's sprite atlas with it. + cx.on_release(|this, cx| { + let images = this.local.borrow_mut().prune(None); + free_images(images, cx); + }) + .detach(); let mut tasks = Vec::new(); let load = match engine { Some(engine) => { tasks.push(cx.spawn(async move |this, cx| { - let script = crate::attachments::read_attachment_bytes( - &engine, - cx.background_executor(), - target_device_id.as_deref(), - &path, - None, - ) - .await - .and_then(|(_, _, bytes)| String::from_utf8(bytes).ok()); + let read = |path: String| { + let (engine, target) = (engine.clone(), target_device_id.clone()); + let executor = cx.background_executor().clone(); + async move { + crate::attachments::read_attachment_bytes( + &engine, + &executor, + target.as_deref(), + &path, + None, + ) + .await + .map(|(_, _, bytes)| bytes) + } + }; + let script = read(path.clone()) + .await + .and_then(|bytes| String::from_utf8(bytes).ok()); + // Its files sit beside it, listed in `{id}-assets.json` + // (absent for a visual without any). + let mut files = HashMap::new(); + if script.is_some() + && let Some(stem) = path.strip_suffix("-visual.luau") + && let Some(names) = read(format!("{stem}-assets.json")) + .await + .and_then(|bytes| serde_json::from_slice::>(&bytes).ok()) + { + for name in names + .into_iter() + .filter(|name| zeron_visual::assets::valid_name(name)) + .take(zeron_visual::assets::MAX_ASSETS) + { + if let Some(bytes) = read(format!("{stem}-asset-{name}")).await { + files.insert(name, Arc::<[u8]>::from(bytes)); + } + } + } + let assets = zeron_visual::Assets::new(files); let _ = this.update(cx, |this, cx| match script { - Some(script) => this.start(&script, cx), + Some(script) => this.start(&script, assets, cx), None => { this.load = Load::Failed("Visual unavailable".into()); cx.notify(); @@ -204,13 +241,14 @@ impl VisualFrame { /// Run a script's setup and first render (the engine already ran the /// same script before publishing it, so failures here are rare). - fn start(&mut self, script: &str, cx: &mut Context) { + fn start(&mut self, script: &str, assets: zeron_visual::Assets, cx: &mut Context) { // The row id ends with the part's visual id, which the engine // seeded its check with: the reader sees the same random data. let visual_id = self.id.rsplit('#').next().unwrap_or(&self.id); let seed = zeron_visual::seed_for(visual_id); match zeron_visual::Visual::load(script, seed) { Ok(visual) => { + let visual = visual.with_assets(assets); self.runtime = Some(Rc::new(RefCell::new(visual))); self.load = Load::Ready; self.rerender(cx); @@ -231,6 +269,13 @@ impl VisualFrame { match rendered { Ok(tree) => { let changed = self.tree.as_deref() != Some(&tree); + if changed { + // Nodes that left the tree take their drawings with them. + let mut keep = std::collections::HashSet::new(); + Self::node_keys(&tree, &self.id, &mut keep); + let images = self.local.borrow_mut().prune(Some(&keep)); + free_images(images, cx); + } self.tree = Some(Rc::new(tree)); self.generation += 1; self.error = None; @@ -364,6 +409,7 @@ impl Palette { Token::Danger => self.danger, Token::Border => self.border, Token::Chart(i) => self.chart[usize::from(i) % 6], + Token::Rgb(rgb) => gpui::rgb(rgb).into(), } } @@ -476,7 +522,65 @@ fn status_line(message: SharedString, color: Hsla) -> AnyElement { .into_any_element() } +impl Local { + /// Forget the drawings of nodes not in `keep` (all of them, without + /// one), returning their images to free. + fn prune( + &mut self, + keep: Option<&std::collections::HashSet>, + ) -> Vec> { + let kept = |key: &SharedString| keep.is_some_and(|keep| keep.contains(key)); + let mut images = Vec::new(); + self.shaders.retain(|key, cache| { + kept(key) || { + images.push(cache.image.clone()); + false + } + }); + self.scenes.retain(|key, cache| { + kept(key) || { + images.extend(cache.image.as_ref().map(|(_, image)| image.image.clone())); + false + } + }); + self.canvases.retain(|key, _| kept(key)); + self.cameras.retain(|key, _| kept(key)); + images + } +} + +fn free_images(images: Vec>, cx: &mut App) { + if !images.is_empty() { + cx.defer(move |cx| { + for image in images { + cx.drop_image(image, None); + } + }); + } +} + impl VisualFrame { + /// Every node's key, as [`VisualFrame::node`] walks the tree from `path`. + fn node_keys(node: &Node, path: &str, out: &mut std::collections::HashSet) { + out.insert(Self::key(node, path)); + let children: &[Node] = match &node.kind { + NodeKind::Stack(stack) => &stack.children, + NodeKind::Grid { children, .. } => children, + NodeKind::Table(table) => { + for (r, row) in table.rows.iter().enumerate() { + for (c, cell) in row.iter().enumerate() { + Self::node_keys(cell, &format!("{path}.{r}.{c}"), out); + } + } + &[] + } + _ => &[], + }; + for (ix, child) in children.iter().enumerate() { + Self::node_keys(child, &format!("{path}.{ix}"), out); + } + } + fn key(node: &Node, path: &str) -> SharedString { match &node.key { Some(key) => format!("key:{key}").into(), @@ -1479,6 +1583,32 @@ impl VisualFrame { .w_full() .h(px(node.height)); el = self.pointer_tracking(el, &key, cx); + let (down_key, up_key, out_key) = (key.clone(), key.clone(), key.clone()); + el = el + .on_mouse_down( + MouseButton::Left, + cx.listener(move |this, event: &gpui::MouseDownEvent, _, cx| { + this.local + .borrow_mut() + .presses + .insert(down_key.clone(), event.position); + cx.notify(); + }), + ) + .on_mouse_up( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + this.local.borrow_mut().presses.remove(&up_key); + cx.notify(); + }), + ) + .on_mouse_up_out( + MouseButton::Left, + cx.listener(move |this, _: &gpui::MouseUpEvent, _, cx| { + this.local.borrow_mut().presses.remove(&out_key); + cx.notify(); + }), + ); if node.orbit { el = self.orbit_handlers(el, &key, spin, cx); } @@ -1487,6 +1617,17 @@ impl VisualFrame { for (i, (_, value)) in node.inputs.iter().enumerate() { inputs[i / 4][i % 4] = *value; } + let assets = self + .runtime + .as_ref() + .map(|runtime| runtime.borrow().assets().clone()) + .unwrap_or_default(); + let textures: Vec = node + .textures + .iter() + // A missing one keeps its slot (and samples as clear). + .map(|name| zeron_visual::TextureData::Encoded(assets.get(name).unwrap_or_else(|| Arc::from([])))) + .collect(); let (code, buffers, blend, bloom, zoom) = ( node.code.clone(), node.buffers.clone(), @@ -1497,7 +1638,6 @@ impl VisualFrame { // Buffers evolve every frame, so a shader with any keeps running. let moving = moving || !buffers.is_empty() && !cx.reduce_motion(); let (local, time) = (self.local.clone(), self.time()); - let drag = orbit.drag; el.child( canvas( move |bounds, window, _| { @@ -1510,15 +1650,16 @@ impl VisualFrame { (f32::from(bounds.size.height) * scale).round().max(1.0) as u32, ); let mut state = local.borrow_mut(); - // Shadertoy's mouse: pixels from the bottom-left; xy while - // pressed and zw where it went down (negative after). + // Shadertoy's mouse: pixels from the bottom-left; xy where + // the pointer is, zw where it went down while it's held + // (negative otherwise). let device = |p: Point| { let at = p - bounds.origin; [f32::from(at.x) * scale, size.1 as f32 - f32::from(at.y) * scale] }; - let mouse = match (drag, state.pointers.get(&key)) { - (Some((start, _, _)), Some(now)) => { - let ([x, y], [sx, sy]) = (device(*now), device(start)); + let mouse = match (state.presses.get(&key), state.pointers.get(&key)) { + (Some(start), Some(now)) => { + let ([x, y], [sx, sy]) = (device(*now), device(*start)); [x, y, sx, sy] } (None, Some(now)) => { @@ -1529,10 +1670,11 @@ impl VisualFrame { }; let buffers_run = !buffers.is_empty(); let previous = state.shaders.get(&key); + // A still shader keeps the time it first showed. let shown_time = if animate { time } else { - previous.map_or(0.0, |p| p.last_time) + previous.map_or(0.0, |p| f64::from(p.uniforms.time[0])) }; let frame = previous.map_or(0, |p| p.frame + 1); let delta = previous.map_or(0.0, |p| (time - p.last_time).max(0.0)); @@ -1563,6 +1705,7 @@ impl VisualFrame { inputs: &names, blend, bloom, + textures: &textures, }, &uniforms, size.0, @@ -1721,7 +1864,9 @@ impl VisualFrame { .get(&key) .and_then(|c| c.final_cost) .is_some_and(|cost| cost < std::time::Duration::from_millis(8)); - let quality = if (moving || dragging) && !cheap { + // Quality only steers the software renderer; on the GPU + // a finished drag mustn't render the same frame again. + let quality = if (moving || dragging) && !cheap && !zeron_visual_gpu::available() { zeron_visual::Quality::Draft } else { zeron_visual::Quality::Final diff --git a/crates/visual-gpu/Cargo.toml b/crates/visual-gpu/Cargo.toml index 5cbc6c9690..8998230a03 100644 --- a/crates/visual-gpu/Cargo.toml +++ b/crates/visual-gpu/Cargo.toml @@ -12,3 +12,7 @@ zeron-visual.workspace = true wgpu = { version = "29", features = ["glsl"] } pollster = "0.4" tracing.workspace = true + +[dev-dependencies] +# Test textures and environment maps. +image = { version = "0.25.10", default-features = false, features = ["png"] } diff --git a/crates/visual-gpu/examples/render_shader.rs b/crates/visual-gpu/examples/render_shader.rs index d399144b7f..e68d4383a7 100644 --- a/crates/visual-gpu/examples/render_shader.rs +++ b/crates/visual-gpu/examples/render_shader.rs @@ -58,6 +58,7 @@ fn main() { inputs: &names, blend, bloom, + textures: &[], }; let frame = render_shader(&spec, &uniforms, w, h, &mut None).expect("render"); let mut ppm = format!("P6\n{w} {h}\n255\n").into_bytes(); diff --git a/crates/visual-gpu/src/lib.rs b/crates/visual-gpu/src/lib.rs index fac8aeca3b..5a31e29b66 100644 --- a/crates/visual-gpu/src/lib.rs +++ b/crates/visual-gpu/src/lib.rs @@ -11,6 +11,7 @@ use std::sync::OnceLock; mod scene; mod shader; +mod textures; pub use scene::render_scene; pub use shader::{ShaderSpec, ShaderState, render_shader}; @@ -28,19 +29,47 @@ pub(crate) struct Gpu { pub(crate) queue: wgpu::Queue, pub(crate) shaders: OnceLock, pub(crate) scene: OnceLock, + pub(crate) textures: textures::Textures, } /// The shared device, made on first use; `None` without a usable adapter. pub(crate) fn gpu() -> Option<&'static Gpu> { - static GPU: OnceLock> = OnceLock::new(); - GPU.get_or_init(|| match pollster::block_on(Gpu::new()) { - Ok(gpu) => Some(gpu), - Err(error) => { - tracing::warn!(%error, "visual GPU unavailable"); - None - } + device().as_ref().ok() +} + +fn device() -> &'static Result { + static GPU: OnceLock> = OnceLock::new(); + GPU.get_or_init(|| { + pollster::block_on(Gpu::new()) + .inspect_err(|error| tracing::warn!(%error, "visual GPU unavailable")) }) - .as_ref() +} + +/// Run `f`, catching every error wgpu reports while it runs (validation, +/// internal, out of memory) as this call's error. +pub(crate) fn guarded(device: &wgpu::Device, f: impl FnOnce() -> T) -> Result { + let scopes = [ + wgpu::ErrorFilter::OutOfMemory, + wgpu::ErrorFilter::Internal, + wgpu::ErrorFilter::Validation, + ] + .map(|filter| device.push_error_scope(filter)); + let out = f(); + let mut error = None; + for scope in scopes.into_iter().rev() { + if let Some(e) = pollster::block_on(scope.pop()) { + error.get_or_insert_with(|| e.to_string()); + } + } + match error { + Some(error) => Err(error), + None => Ok(out), + } +} + +/// Why visuals can't use the GPU here, if they can't. +pub fn unavailable_reason() -> Option { + device().as_ref().err().cloned() } /// Whether visuals can render on the GPU here. @@ -51,6 +80,23 @@ pub fn available() -> bool { /// The target every visual renders into before readback. pub(crate) const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; +/// WGSL for the final passes: how much of a pixel survives near the frame's +/// edges, so light running off a visual fades out instead of stopping at a +/// hard line. The ramp is 15% of the shorter side, eased at both ends +/// (smootherstep) so neither the start nor the end of the fade shows. +pub(crate) const EDGE_FADE: &str = r#" +fn edge_ease(d: f32, width: f32) -> f32 { + let t = clamp(d / width, 0.0, 1.0); + return t * t * t * (t * (t * 6.0 - 15.0) + 10.0); +} + +fn edge_fade(pixel: vec2, size: vec2) -> f32 { + let width = max(0.15 * min(size.x, size.y), 1.0); + let d = min(pixel, size - pixel); + return edge_ease(d.x, width) * edge_ease(d.y, width); +} +"#; + impl Gpu { async fn new() -> Result { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { @@ -71,19 +117,29 @@ impl Gpu { let (device, queue) = adapter .request_device(&wgpu::DeviceDescriptor { label: Some("zeron visuals"), + // What this adapter has, not wgpu's defaults: some devices + // (the iOS simulator) fall short of a default limit the + // renderers never come near. + required_limits: adapter.limits(), ..Default::default() }) .await .map_err(|e| e.to_string())?; + // wgpu panics on an error nothing catches; a visual's mistake (or a + // lost device) must cost that visual, not the app. + device.on_uncaptured_error(std::sync::Arc::new(|error| { + tracing::warn!(%error, "visual GPU error"); + })); Ok(Self { device, queue, shaders: OnceLock::new(), scene: OnceLock::new(), + textures: Default::default(), }) } -/// Read a rendered texture back as straight-alpha BGRA. + /// Read a rendered texture back as straight-alpha BGRA. pub(crate) fn read_back( &self, mut encoder: wgpu::CommandEncoder, diff --git a/crates/visual-gpu/src/scene.rs b/crates/visual-gpu/src/scene.rs index cc4e512b5c..493292d07d 100644 --- a/crates/visual-gpu/src/scene.rs +++ b/crates/visual-gpu/src/scene.rs @@ -3,11 +3,15 @@ //! //! Passes, per frame: //! 1. shadow: the key light's depth over the geometry (orthographic); -//! 2. main: opaque surfaces into a multisampled HDR target with depth (GGX -//! specular, sky/ground ambient, key light with PCF shadows, a fill -//! light, glow), then lines and points unlit; resolved for sampling; +//! 2. main: the environment map behind everything, when the scene asks for +//! it; opaque surfaces (textured or not) into a multisampled HDR target +//! with depth (GGX specular, sky/ground ambient, key light with PCF +//! shadows, a fill light, glow); other see-through surfaces blended over +//! them unbent, far to near; sprites (flares, glints) adding their light +//! behind nearer surfaces; then lines and points unlit; resolved; //! 3. glass: transmissive surfaces over that, refracting the resolved image -//! per color channel (dispersion) under a Fresnel reflection; +//! (see-through surfaces and sprites behind it included) per color +//! channel (dispersion) under a Fresnel reflection; //! 4. bloom: bright parts downsampled and blurred; //! 5. composite: bloom added, a fine depth-edge outline, tonemapped to sRGB //! straight alpha, transparent wherever nothing was drawn. @@ -15,7 +19,8 @@ use wgpu::util::DeviceExt; use zeron_visual::{Camera, CameraFrame, Color, Scene, V3}; -use crate::{FORMAT, Frame, gpu}; +use crate::textures::{Loaded, Space}; +use crate::{EDGE_FADE, FORMAT, Frame, gpu}; const HDR: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; const DEPTH: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float; @@ -33,9 +38,10 @@ struct Vertex { color: [f32; 4], /// Roughness, metalness, glow, glass. mat: [f32; 4], - /// Ior, dispersion, two-sided, kind (0 surface, 1 line, 2 point disc). + /// Ior, dispersion, two-sided, kind (0 surface, 1 line, 2 point disc, + /// 3 sprite). opt: [f32; 4], - /// Point-disc coordinates in -1..1. + /// A surface's texture coordinates; a disc's or sprite's in -1..1. uv: [f32; 2], } @@ -72,6 +78,10 @@ struct Globals { right: [f32; 4], up: [f32; 4], forward: [f32; 4], + /// The environment map: intensity, present, drawn behind, mip levels. + env: [f32; 4], + /// The projection's x and y scale, to turn screen points into rays. + lens: [f32; 4], } fn bytes(value: &T) -> &[u8] { @@ -90,17 +100,26 @@ fn slice_bytes(values: &[T]) -> &[u8] { pub(crate) struct Pipelines { globals_layout: wgpu::BindGroupLayout, + material_layout: wgpu::BindGroupLayout, glass_layout: wgpu::BindGroupLayout, post_layout: wgpu::BindGroupLayout, shadow: wgpu::RenderPipeline, opaque: wgpu::RenderPipeline, overlay: wgpu::RenderPipeline, glass: wgpu::RenderPipeline, + translucent: wgpu::RenderPipeline, + background: wgpu::RenderPipeline, + sprite: wgpu::RenderPipeline, bright: wgpu::RenderPipeline, blur: wgpu::RenderPipeline, composite: wgpu::RenderPipeline, linear: wgpu::Sampler, shadow_sampler: wgpu::Sampler, + /// Textures repeat and filter across mips, anisotropically. + material_sampler: wgpu::Sampler, + /// Equirectangular: wraps around, clamps at the poles. + env_sampler: wgpu::Sampler, + white: wgpu::TextureView, } const SURFACE: &str = r#" @@ -116,10 +135,16 @@ struct Globals { right: vec4, up: vec4, forward: vec4, + env: vec4, + lens: vec4, }; @group(0) @binding(0) var g: Globals; @group(0) @binding(1) var shadow_map: texture_depth_2d; @group(0) @binding(2) var shadow_sampler: sampler_comparison; +@group(0) @binding(3) var env_map: texture_2d; +@group(0) @binding(4) var env_sampler: sampler; +@group(1) @binding(0) var material_map: texture_2d; +@group(1) @binding(1) var material_sampler: sampler; struct In { @location(0) pos: vec3, @@ -191,10 +216,35 @@ fn fresnel(cos_t: f32, f0: vec3) -> vec3 { return f0 + (1.0 - f0) * pow(1.0 - cos_t, 5.0); } -fn environment(dir: vec3) -> vec3 { +fn env_uv(dir: vec3) -> vec2 { + let d = normalize(dir); + return vec2(atan2(d.x, -d.z) / (2.0 * PI) + 0.5, acos(clamp(d.y, -1.0, 1.0)) / PI); +} + +// The light arriving from `dir`, as blurred as a surface of roughness +// `rough` sees it: the environment map when there is one, else a sky over +// a ground. +fn environment_at(dir: vec3, rough: f32) -> vec3 { + if (g.env.y > 0.5) { + let level = clamp(rough, 0.0, 1.0) * max(g.env.w - 3.0, 0.0); + return textureSampleLevel(env_map, env_sampler, env_uv(dir), level).rgb * g.env.x; + } return mix(g.ground.rgb, g.sky.rgb, smoothstep(-0.3, 0.6, dir.y)); } +fn environment(dir: vec3) -> vec3 { + return environment_at(dir, 0.0); +} + +// Diffuse light from around `n`: the map's coarsest useful mip. +fn irradiance(n: vec3) -> vec3 { + if (g.env.y > 0.5) { + let level = max(g.env.w - 3.0, 0.0); + return textureSampleLevel(env_map, env_sampler, env_uv(n), level).rgb * g.env.x; + } + return environment(n); +} + // The shading normal turned toward the camera; w is 0 on the far side of a // closed solid, which is not drawn. fn facing(i: Out, v: vec3) -> vec4 { @@ -204,8 +254,7 @@ fn facing(i: Out, v: vec3) -> vec4 { return vec4(n, 1.0); } -fn shade(i: Out, n: vec3, v: vec3) -> vec3 { - let albedo = i.color.rgb; +fn shade(i: Out, albedo: vec3, n: vec3, v: vec3) -> vec3 { let rough = clamp(i.mat.x, 0.04, 1.0); let metal = i.mat.y; let l = normalize(g.light.xyz); @@ -222,16 +271,62 @@ fn shade(i: Out, n: vec3, v: vec3) -> vec3 { let fill = kd * albedo / PI * max(dot(n, fill_dir), 0.0) * 0.9; let r = reflect(-v, n); let env_f = fresnel(n_v, f0); - let ambient = kd * albedo * environment(n) * 0.85 + var ambient = kd * albedo * environment(n) * 0.85 + env_f * environment(r) * (1.0 - rough * 0.85) * 0.8; + if (g.env.y > 0.5) { + ambient = kd * albedo * irradiance(n) + env_f * environment_at(r, rough); + } return key + fill + ambient + albedo * i.mat.z; } @fragment fn fs_opaque(i: Out) -> @location(0) vec4 { + // Sampled first, while every fragment is still running (derivatives). + let tex = textureSample(material_map, material_sampler, i.uv); let v = normalize(g.eye.xyz - i.pos); let f = facing(i, v); - if (f.w == 0.0) { discard; } - return vec4(shade(i, f.xyz, v), 1.0); + // A texture's clear parts are cut out. + if (f.w == 0.0 || tex.a < 0.5) { discard; } + return vec4(shade(i, i.color.rgb * tex.rgb, f.xyz, v), 1.0); +} + +// A see-through surface that isn't glass: lit like a solid one and laid over +// what's behind it, unbent (premultiplied). +@fragment fn fs_translucent(i: Out) -> @location(0) vec4 { + let tex = textureSample(material_map, material_sampler, i.uv); + let v = normalize(g.eye.xyz - i.pos); + let f = facing(i, v); + let a = clamp(i.color.a * tex.a, 0.0, 1.0); + if (f.w == 0.0 || a <= 0.0) { discard; } + return vec4(shade(i, i.color.rgb * tex.rgb, f.xyz, v) * a, a); +} + +struct Screen { @builtin(position) clip: vec4, @location(0) ndc: vec2 }; + +@vertex fn vs_background(@builtin(vertex_index) k: u32) -> Screen { + let p = vec2(f32((k << 1u) & 2u), f32(k & 2u)) * 2.0 - 1.0; + var o: Screen; + o.clip = vec4(p, 0.5, 1.0); + o.ndc = p; + return o; +} + +// The environment map seen straight on, behind the scene. +@fragment fn fs_background(i: Screen) -> @location(0) vec4 { + let dir = g.forward.xyz + g.right.xyz * (i.ndc.x / g.lens.x) + g.up.xyz * (i.ndc.y / g.lens.y); + return vec4(environment(dir), 1.0); +} + +// A sprite's light: its image, or a soft round glow (mat.x 0). +@fragment fn fs_sprite(i: Out) -> @location(0) vec4 { + let tex = textureSample(material_map, material_sampler, vec2(i.uv.x * 0.5 + 0.5, 0.5 - i.uv.y * 0.5)); + let d2 = dot(i.uv, i.uv); + var shape = tex.rgb * tex.a; + if (i.mat.x < 0.5) { + let soft = exp(-d2 * 5.0) * 0.7 + exp(-d2 * 48.0) * 0.8; + shape = vec3(soft * (1.0 - smoothstep(0.7, 1.0, d2))); + } + let c = shape * i.color.rgb * i.color.a * i.mat.z; + return vec4(c, clamp(max(c.r, max(c.g, c.b)), 0.0, 1.0)); } @fragment fn fs_overlay(i: Out) -> @location(0) vec4 { @@ -242,10 +337,11 @@ fn shade(i: Out, n: vec3, v: vec3) -> vec3 { "#; const GLASS: &str = r#" -@group(1) @binding(0) var behind: texture_2d; -@group(1) @binding(1) var behind_sampler: sampler; +@group(2) @binding(0) var behind: texture_2d; +@group(2) @binding(1) var behind_sampler: sampler; @fragment fn fs_glass(i: Out) -> @location(0) vec4 { + let tex = textureSample(material_map, material_sampler, i.uv); let v = normalize(g.eye.xyz - i.pos); let f = facing(i, v); if (f.w == 0.0) { discard; } @@ -253,19 +349,33 @@ const GLASS: &str = r#" let ior = max(i.opt.x, 1.0); let spread = i.opt.y * 0.09; let screen = i.clip.xy * g.viewport.zw; + let glass = clamp(i.mat.w, 0.0, 1.0); // Bend what's behind by the surface's tilt across the view, a little - // more for each shorter wavelength. + // more for each shorter wavelength; partial glass bends less. let view_n = vec2(dot(n, g.right.xyz), -dot(n, g.up.xyz)); var seen = vec3(0.0); var cover = 0.0; let iors = vec3(ior - spread, ior, ior + spread); for (var c = 0; c < 3; c++) { - let bend = (1.0 - 1.0 / iors[c]) * 0.34; + let bend = (1.0 - 1.0 / iors[c]) * 0.34 * glass; let s = textureSampleLevel(behind, behind_sampler, screen - view_n * bend, 0.0); seen[c] = s[c]; cover = max(cover, s.a); } - let tint = mix(vec3(1.0), i.color.rgb, 0.35); + let albedo = i.color.rgb * tex.rgb; + // With an environment, what isn't covered behind shows the map, bent + // through the glass per channel. + if (g.env.y > 0.5) { + var bent = vec3(0.0); + for (var c = 0; c < 3; c++) { + var t = refract(-v, n, 1.0 / iors[c]); + if (dot(t, t) < 1e-4) { t = reflect(-v, n); } + bent[c] = environment(t)[c]; + } + seen = seen + bent * (1.0 - cover); + cover = 1.0; + } + let tint = mix(vec3(1.0), albedo, 0.35); let f0 = pow((ior - 1.0) / (ior + 1.0), 2.0); let n_v = max(dot(n, v), 0.0); let fr = f0 + (1.0 - f0) * pow(1.0 - n_v, 5.0); @@ -274,13 +384,12 @@ const GLASS: &str = r#" let glint = pow(max(dot(r, l), 0.0), 220.0) * 6.0; let reflection = environment(r) * 1.2 + vec3(glint); let through = seen * tint; - let glass = clamp(i.mat.w, 0.0, 1.0); // Where nothing stands behind the glass, it shows its reflection and a // faint body tint over the chat. let body = 0.12 + 0.25 * (1.0 - glass); let alpha = clamp(max(cover, fr + body + glint), 0.0, 1.0); - let color = through + (reflection - through) * fr + i.color.rgb * body * (1.0 - cover) - + i.color.rgb * i.mat.z; + let color = through + (reflection - through) * fr + albedo * body * (1.0 - cover) + + albedo * i.mat.z; return vec4(color, alpha); } "#; @@ -354,12 +463,13 @@ const NEIGHBOURS = array, 4>(vec2(1, 0), vec2(-1, 0), vec2(0.0); } var straight = (c.rgb + glow) / alpha; straight = straight * (1.0 - 0.18 * edge); - return vec4(to_srgb(aces(straight)), alpha); + let fade = edge_fade(i.clip.xy, vec2(size)); + return vec4(to_srgb(aces(straight)), alpha * fade); } "#; impl Pipelines { - fn new(device: &wgpu::Device) -> Self { + fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Self { let uniform = |binding, visibility| wgpu::BindGroupLayoutEntry { binding, visibility, @@ -393,6 +503,15 @@ impl Pipelines { uniform(0, wgpu::ShaderStages::VERTEX_FRAGMENT), texture(1, wgpu::TextureSampleType::Depth, false), sampler(2, wgpu::SamplerBindingType::Comparison), + texture(3, float, false), + sampler(4, wgpu::SamplerBindingType::Filtering), + ], + }); + let material_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("scene material"), + entries: &[ + texture(0, float, false), + sampler(1, wgpu::SamplerBindingType::Filtering), ], }); let glass_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { @@ -422,7 +541,7 @@ impl Pipelines { }); let post = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("scene post"), - source: wgpu::ShaderSource::Wgsl(POST.into()), + source: wgpu::ShaderSource::Wgsl(format!("{EDGE_FADE}{POST}").into()), }); let layout = |layouts: &[Option<&wgpu::BindGroupLayout>]| { device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { @@ -431,8 +550,13 @@ impl Pipelines { immediate_size: 0, }) }; - let surface_layout = layout(&[Some(&globals_layout)]); - let glass_pipeline_layout = layout(&[Some(&globals_layout), Some(&glass_layout)]); + let shadow_layout = layout(&[Some(&globals_layout)]); + let surface_layout = layout(&[Some(&globals_layout), Some(&material_layout)]); + let glass_pipeline_layout = layout(&[ + Some(&globals_layout), + Some(&material_layout), + Some(&glass_layout), + ]); let post_pipeline_layout = layout(&[Some(&post_layout)]); let msaa = wgpu::MultisampleState { count: SAMPLES, @@ -447,38 +571,39 @@ impl Pipelines { bias, }) }; - let surface_pipeline = |label: &str, - module: &wgpu::ShaderModule, - layout: &wgpu::PipelineLayout, - fragment: &str, - blend: Option, - depth_stencil: Option| { - device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some(label), - layout: Some(layout), - vertex: wgpu::VertexState { - module, - entry_point: Some("vs"), - compilation_options: Default::default(), - buffers: &[Vertex::layout()], - }, - primitive: wgpu::PrimitiveState::default(), - depth_stencil, - multisample: msaa, - fragment: Some(wgpu::FragmentState { - module, - entry_point: Some(fragment), - compilation_options: Default::default(), - targets: &[Some(wgpu::ColorTargetState { - format: HDR, - blend, - write_mask: wgpu::ColorWrites::ALL, - })], - }), - multiview_mask: None, - cache: None, - }) - }; + let surface_pipeline = + |label: &str, + module: &wgpu::ShaderModule, + layout: &wgpu::PipelineLayout, + fragment: &str, + blend: Option, + depth_stencil: Option| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(layout), + vertex: wgpu::VertexState { + module, + entry_point: Some("vs"), + compilation_options: Default::default(), + buffers: &[Vertex::layout()], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil, + multisample: msaa, + fragment: Some(wgpu::FragmentState { + module, + entry_point: Some(fragment), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: HDR, + blend, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }; let opaque = surface_pipeline( "scene opaque", &surface, @@ -510,9 +635,68 @@ impl Pipelines { None, depth(true, Default::default()), ); + // Drawn far to near, so it tests depth but doesn't write it. + let translucent = surface_pipeline( + "scene translucent", + &surface, + &surface_layout, + "fs_translucent", + Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + depth(false, Default::default()), + ); + let sprite = surface_pipeline( + "scene sprites", + &surface, + &surface_layout, + "fs_sprite", + Some(wgpu::BlendState { + color: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::One, + dst_factor: wgpu::BlendFactor::One, + operation: wgpu::BlendOperation::Add, + }, + alpha: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::One, + dst_factor: wgpu::BlendFactor::One, + operation: wgpu::BlendOperation::Add, + }, + }), + depth(false, Default::default()), + ); + let background = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("scene environment"), + layout: Some(&surface_layout), + vertex: wgpu::VertexState { + module: &surface, + entry_point: Some("vs_background"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: Some(wgpu::DepthStencilState { + format: DEPTH, + depth_write_enabled: Some(false), + depth_compare: Some(wgpu::CompareFunction::Always), + stencil: Default::default(), + bias: Default::default(), + }), + multisample: msaa, + fragment: Some(wgpu::FragmentState { + module: &surface, + entry_point: Some("fs_background"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: HDR, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); let shadow = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("scene shadow"), - layout: Some(&surface_layout), + layout: Some(&shadow_layout), vertex: wgpu::VertexState { module: &surface, entry_point: Some("vs_shadow"), @@ -568,12 +752,36 @@ impl Pipelines { blur: post_pipeline("scene bloom blur", "fs_blur", HDR), composite: post_pipeline("scene composite", "fs_composite", FORMAT), globals_layout, + material_layout, glass_layout, post_layout, shadow, opaque, overlay, glass, + translucent, + background, + sprite, + material_sampler: device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("scene material"), + address_mode_u: wgpu::AddressMode::Repeat, + address_mode_v: wgpu::AddressMode::Repeat, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::MipmapFilterMode::Linear, + anisotropy_clamp: 8, + ..Default::default() + }), + env_sampler: device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("scene environment"), + address_mode_u: wgpu::AddressMode::Repeat, + address_mode_v: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::MipmapFilterMode::Linear, + ..Default::default() + }), + white: crate::textures::solid(device, queue, [255; 4]), linear: device.create_sampler(&wgpu::SamplerDescriptor { label: Some("scene linear"), mag_filter: wgpu::FilterMode::Linear, @@ -653,27 +861,72 @@ fn columns(rows: [[f32; 4]; 4]) -> [[f32; 4]; 4] { /// toward the viewer's default angle and ground the geometry. const KEY_LIGHT: V3 = [-0.45, 0.85, 0.35]; +/// A run of vertices drawn with one texture (`None`: untextured). +#[derive(Debug, Clone, PartialEq)] +struct Batch { + texture: Option, + range: std::ops::Range, +} + +/// Append `vertices` to `out`, as part of the last batch when it shares +/// the texture. +fn push_batch( + out: &mut Vec, + batches: &mut Vec, + texture: Option, + vertices: &[Vertex], +) { + let start = out.len() as u32; + out.extend_from_slice(vertices); + let end = out.len() as u32; + match batches.last_mut() { + Some(last) if last.texture == texture && last.range.end == start => last.range.end = end, + _ => batches.push(Batch { + texture, + range: start..end, + }), + } +} + struct Geometry { /// Opaque surfaces; the first `shadowed` vertices cast shadows (glowing /// surfaces, which are lights, come after and don't). opaque: Vec, + opaque_batches: Vec, shadowed: usize, glass: Vec, + glass_batches: Vec, + /// See-through surfaces that aren't glass, far to near. + translucent: Vec, + translucent_batches: Vec, overlay: Vec, + sprites: Vec, + sprite_batches: Vec, } fn geometry(scene: &Scene, frame: &CameraFrame, rgba: &dyn Fn(Color) -> [f32; 4]) -> Geometry { - let mut glowing = Vec::new(); + use std::collections::BTreeMap; + let mut lit: BTreeMap, Vec> = BTreeMap::new(); + let mut glowing: BTreeMap, Vec> = BTreeMap::new(); let mut out = Geometry { opaque: Vec::new(), + opaque_batches: Vec::new(), shadowed: 0, glass: Vec::new(), + glass_batches: Vec::new(), + translucent: Vec::new(), + translucent_batches: Vec::new(), overlay: Vec::new(), + sprites: Vec::new(), + sprite_batches: Vec::new(), }; - let mut glass_faces: Vec<(f32, Vec)> = Vec::new(); + let mut glass_faces: Vec<(f32, Option, Vec)> = Vec::new(); + let mut translucent_faces: Vec<(f32, Option, Vec)> = Vec::new(); for face in scene.face_views() { let m = face.material; let color = linear(rgba(face.color)); + // A texture needs coordinates to be sampled by. + let texture = face.texture.filter(|_| face.uvs.is_some()); let vertex = |i: usize| Vertex { pos: face.corners[i], normal: face.normals.map_or(face.normal, |n| n[i]), @@ -686,7 +939,7 @@ fn geometry(scene: &Scene, frame: &CameraFrame, rgba: &dyn Fn(Color) -> [f32; 4] if face.two_sided { 1.0 } else { 0.0 }, 0.0, ], - uv: [0.0; 2], + uv: face.uvs.map_or([0.0; 2], |uv| uv[i]), }; let mut tris = Vec::with_capacity((face.corners.len() - 2) * 3); for i in 1..face.corners.len() - 1 { @@ -695,18 +948,40 @@ fn geometry(scene: &Scene, frame: &CameraFrame, rgba: &dyn Fn(Color) -> [f32; 4] if m.glass > 0.0 || color[3] < 0.999 { let centroid = face.corners.iter().fold([0.0; 3], |acc, c| add(acc, *c)); let centroid = scale(centroid, 1.0 / face.corners.len() as f32); - glass_faces.push((dot(sub(centroid, frame.eye), frame.forward), tris)); + let distance = dot(sub(centroid, frame.eye), frame.forward); + // Only glass refracts; a faded wall or pane is just see-through. + if m.glass > 0.0 { + glass_faces.push((distance, texture, tris)); + } else { + translucent_faces.push((distance, texture, tris)); + } } else if m.glow > 0.0 { - glowing.extend(tris); + glowing.entry(texture).or_default().extend(tris); } else { - out.opaque.extend(tris); + lit.entry(texture).or_default().extend(tris); } } + for (texture, tris) in &lit { + push_batch(&mut out.opaque, &mut out.opaque_batches, *texture, tris); + } out.shadowed = out.opaque.len(); - out.opaque.extend(glowing); + for (texture, tris) in &glowing { + push_batch(&mut out.opaque, &mut out.opaque_batches, *texture, tris); + } // Glass far to near, so nearer panes cover farther ones. glass_faces.sort_by(|a, b| b.0.total_cmp(&a.0)); - out.glass = glass_faces.into_iter().flat_map(|(_, t)| t).collect(); + for (_, texture, tris) in &glass_faces { + push_batch(&mut out.glass, &mut out.glass_batches, *texture, tris); + } + translucent_faces.sort_by(|a, b| b.0.total_cmp(&a.0)); + for (_, texture, tris) in &translucent_faces { + push_batch( + &mut out.translucent, + &mut out.translucent_batches, + *texture, + tris, + ); + } // Lines and points face the camera, sized in screen pixels. let world_per_pixel = @@ -743,9 +1018,42 @@ fn geometry(scene: &Scene, frame: &CameraFrame, rgba: &dyn Fn(Color) -> [f32; 4] (sub(sub(at, x), y), [-1.0, -1.0]), ]; for i in [0, 1, 2, 0, 2, 3] { - out.overlay.push(flat(corners[i].0, color, 2.0, corners[i].1)); + out.overlay + .push(flat(corners[i].0, color, 2.0, corners[i].1)); } } + let mut sprites: Vec<_> = scene.sprites().iter().collect(); + sprites.sort_by_key(|sprite| sprite.texture); + for sprite in sprites { + let color = linear(rgba(sprite.color)); + let r = sprite.size * 0.5 * world_per_pixel(sprite.at); + let (x, y) = (scale(frame.right, r), scale(frame.up, r)); + let at = sprite.at; + let corners = [ + (add(sub(at, x), y), [-1.0, 1.0]), + (add(add(at, x), y), [1.0, 1.0]), + (sub(add(at, x), y), [1.0, -1.0]), + (sub(sub(at, x), y), [-1.0, -1.0]), + ]; + let quad: Vec = [0, 1, 2, 0, 2, 3] + .into_iter() + .map(|i| Vertex { + mat: [ + if sprite.texture.is_some() { 1.0 } else { 0.0 }, + 0.0, + sprite.glow, + 0.0, + ], + ..flat(corners[i].0, color, 3.0, corners[i].1) + }) + .collect(); + push_batch( + &mut out.sprites, + &mut out.sprite_batches, + sprite.texture, + &quad, + ); + } out } @@ -766,9 +1074,24 @@ pub fn render_scene( (width * pixel_scale).round().max(1.0) as u32, (height * pixel_scale).round().max(1.0) as u32, ); - let pipelines = gpu.scene.get_or_init(|| Pipelines::new(&gpu.device)); + let pipelines = gpu + .scene + .get_or_init(|| Pipelines::new(&gpu.device, &gpu.queue)); let device = &gpu.device; let geometry = geometry(scene, &frame, rgba); + // Textures, decoded and uploaded once each; one that fails draws plain. + let loaded: Vec> = scene + .textures() + .iter() + .map(|t| { + gpu.textures + .get(device, &gpu.queue, &t.data, Space::Color) + .ok() + }) + .collect(); + let environment = scene + .environment() + .and_then(|e| Some((e, loaded.get(e.texture as usize)?.clone()?))); // Camera: the same framing and projection as the software renderer. let distance = dot(sub(frame.center, frame.eye), sub(frame.center, frame.eye)).sqrt(); @@ -831,6 +1154,16 @@ pub fn render_scene( right: [frame.right[0], frame.right[1], frame.right[2], 0.0], up: [frame.up[0], frame.up[1], frame.up[2], 0.0], forward: [f[0], f[1], f[2], 0.0], + env: match &environment { + Some((e, map)) => [ + e.intensity, + 1.0, + if e.background { 1.0 } else { 0.0 }, + map.mips as f32, + ], + None => [0.0; 4], + }, + lens: [sx, sy, 0.0, 0.0], }; let globals_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("scene globals"), @@ -874,6 +1207,36 @@ pub fn render_scene( let output = gpu.target(w, h, "scene output"); let view = |t: &wgpu::Texture| t.create_view(&Default::default()); + let env_view = environment + .as_ref() + .map_or(&pipelines.white, |(_, map)| &map.view); + // Slot 0 is plain white (untextured); texture `i` is slot `i + 1`. + let material_binds: Vec = std::iter::once(Some(&pipelines.white)) + .chain(loaded.iter().map(|l| l.as_ref().map(|l| &l.view))) + .map(|view| { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scene material"), + layout: &pipelines.material_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView( + view.unwrap_or(&pipelines.white), + ), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&pipelines.material_sampler), + }, + ], + }) + }) + .collect(); + let material = |texture: Option| { + &material_binds[texture + .map_or(0, |t| t as usize + 1) + .min(material_binds.len() - 1)] + }; let globals_bind = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("scene globals"), layout: &pipelines.globals_layout, @@ -890,6 +1253,14 @@ pub fn render_scene( binding: 2, resource: wgpu::BindingResource::Sampler(&pipelines.shadow_sampler), }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(env_view), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: wgpu::BindingResource::Sampler(&pipelines.env_sampler), + }, ], }); let vertices = |label: &str, data: &[Vertex]| { @@ -903,13 +1274,25 @@ pub fn render_scene( }; let opaque = vertices("scene opaque", &geometry.opaque); let glass = vertices("scene glass", &geometry.glass); + let translucent = vertices("scene translucent", &geometry.translucent); let overlay = vertices("scene overlay", &geometry.overlay); + let sprites = vertices("scene sprites", &geometry.sprites); let draw = |pass: &mut wgpu::RenderPass, buffer: &Option, count: usize| { if let Some(buffer) = buffer { pass.set_vertex_buffer(0, buffer.slice(..)); pass.draw(0..count as u32, 0..1); } }; + let draw_batches = + |pass: &mut wgpu::RenderPass, buffer: &Option, batches: &[Batch]| { + if let Some(buffer) = buffer { + pass.set_vertex_buffer(0, buffer.slice(..)); + for batch in batches { + pass.set_bind_group(1, material(batch.texture), &[]); + pass.draw(batch.range.clone(), 0..1); + } + } + }; let mut encoder = device.create_command_encoder(&Default::default()); // 1. Shadow. Its bind group (with the sampled map) is unused here: the @@ -930,6 +1313,14 @@ pub fn render_scene( binding: 2, resource: wgpu::BindingResource::Sampler(&pipelines.shadow_sampler), }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(env_view), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: wgpu::BindingResource::Sampler(&pipelines.env_sampler), + }, ], }); { @@ -955,7 +1346,8 @@ pub fn render_scene( } let color_view = view(&msaa_color); let depth_view = view(&msaa_depth); - // 2. Opaque surfaces, then lines and points. + // 2. Opaque surfaces; see-through ones and sprites over them (so glass + // refracts them too); then lines and points. { let resolve = view(&opaque_resolved); let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -982,12 +1374,22 @@ pub fn render_scene( multiview_mask: None, }); pass.set_bind_group(0, &globals_bind, &[]); + pass.set_bind_group(1, material(None), &[]); + if environment.as_ref().is_some_and(|(e, _)| e.background) { + pass.set_pipeline(&pipelines.background); + pass.draw(0..3, 0..1); + } pass.set_pipeline(&pipelines.opaque); - draw(&mut pass, &opaque, geometry.opaque.len()); + draw_batches(&mut pass, &opaque, &geometry.opaque_batches); + pass.set_pipeline(&pipelines.translucent); + draw_batches(&mut pass, &translucent, &geometry.translucent_batches); + pass.set_pipeline(&pipelines.sprite); + draw_batches(&mut pass, &sprites, &geometry.sprite_batches); + pass.set_bind_group(1, material(None), &[]); pass.set_pipeline(&pipelines.overlay); draw(&mut pass, &overlay, geometry.overlay.len()); } - // 3. Glass, refracting the resolved opaque image. + // 3. Glass, refracting everything resolved behind it. let shaded = if glass.is_some() { let behind_bind = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("scene behind glass"), @@ -1028,9 +1430,9 @@ pub fn render_scene( multiview_mask: None, }); pass.set_bind_group(0, &globals_bind, &[]); - pass.set_bind_group(1, &behind_bind, &[]); + pass.set_bind_group(2, &behind_bind, &[]); pass.set_pipeline(&pipelines.glass); - draw(&mut pass, &glass, geometry.glass.len()); + draw_batches(&mut pass, &glass, &geometry.glass_batches); &lit } else { &opaque_resolved @@ -1177,13 +1579,23 @@ mod tests { use zeron_visual::Token; fn palette(color: Color) -> [f32; 4] { - match color.token { - Token::Chart(0) => [0.3, 0.6, 0.95, 1.0], - Token::Chart(1) => [0.95, 0.45, 0.35, 1.0], - Token::Chart(2) => [0.4, 0.85, 0.5, 1.0], - Token::Chart(3) => [0.95, 0.8, 0.3, 1.0], - _ => [0.75, 0.75, 0.78, 1.0], - } + let [r, g, b] = match color.token { + Token::Chart(0) => [0.3, 0.6, 0.95], + Token::Chart(1) => [0.95, 0.45, 0.35], + Token::Chart(2) => [0.4, 0.85, 0.5], + Token::Chart(3) => [0.95, 0.8, 0.3], + _ => [0.75, 0.75, 0.78], + }; + [r, g, b, color.alpha] + } + + fn scene(script: &str) -> Scene { + let mut visual = zeron_visual::Visual::load(script, 1).unwrap(); + let tree = visual.render().unwrap(); + let zeron_visual::NodeKind::Scene(node) = tree.kind else { + panic!("a scene") + }; + visual.draw_scene(node.draw, 0.0).unwrap() } fn room() -> Scene { @@ -1209,6 +1621,104 @@ mod tests { visual.draw_scene(node.draw, 0.0).unwrap() } + fn png(width: u32, height: u32, pixel: impl Fn(u32, u32) -> [u8; 4]) -> std::sync::Arc<[u8]> { + let mut out = Vec::new(); + image::RgbaImage::from_fn(width, height, |x, y| image::Rgba(pixel(x, y))) + .write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png) + .unwrap(); + out.into() + } + + /// A checkered box, a glass prism, a metal ball under a striped sky + /// map, and a flare. + fn gallery() -> Scene { + let assets = zeron_visual::Assets::new( + [ + ( + "checker.png".to_owned(), + png(64, 64, |x, y| { + if (x / 8 + y / 8) % 2 == 0 { + [240, 240, 240, 255] + } else { + [30, 60, 200, 255] + } + }), + ), + ( + "sky.png".to_owned(), + png(256, 128, |x, y| { + let band = (x / 32) % 2 == 0; + let up = 255 - (y * 2).min(255) as u8; + if band { + [255, up, 80, 255] + } else { + [40, 80, up, 255] + } + }), + ), + ] + .into_iter() + .collect(), + ); + let mut visual = zeron_visual::Visual::load( + r##" + return ui.scene { height = 300, draw = function(s) + s:environment { asset = "sky.png", intensity = 1.2, background = true } + s:box { at = { -1.4, 0.5, 0 }, size = 1, texture = "checker.png", roughness = 0.6 } + s:mesh { vertices = { {0.2,0,-0.5}, {1.2,0,-0.5}, {0.7,1.4,-0.5}, {0.2,0,0.5}, {1.2,0,0.5}, {0.7,1.4,0.5} }, + faces = { {1,2,3}, {4,6,5}, {1,4,5,2}, {2,5,6,3}, {3,6,4,1} }, + color = "#ffffff", glass = true, ior = 1.5, dispersion = 1 } + s:sphere { at = { 2.2, 0.6, 0 }, radius = 0.6, color = "#dddddd", metal = 1, roughness = 0.1 } + s:sprite { at = { 0.7, 1.6, 0 }, size = 120, color = "#ffd9a0", glow = 2 } + end } + "##, + 1, + ) + .unwrap() + .with_assets(assets); + let tree = visual.render().unwrap(); + let zeron_visual::NodeKind::Scene(node) = tree.kind else { + panic!("a scene") + }; + visual.draw_scene(node.draw, 0.0).unwrap() + } + + #[test] + fn textures_environments_and_sprites_render() { + if !crate::available() { + return; + } + let scene = gallery(); + let camera = Camera { + yaw: 0.5, + pitch: 0.35, + zoom: 1.0, + }; + let frame = render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + if let Ok(path) = std::env::var("ZERON_VISUAL_DUMP_ASSETS") { + crate::dump_ppm(&frame, &path); + } + // The environment stands behind everything: no clear pixels inside + // the edge fade (15% of the 720 px height), and a faded frame edge. + let (w, h) = (frame.width as usize, frame.height as usize); + let band = 110; + let mut inside = frame.bgra.chunks_exact(4).enumerate().filter(|(i, _)| { + let (x, y) = (i % w, i / w); + (band..w - band).contains(&x) && (band..h - band).contains(&y) + }); + assert!(inside.all(|(_, p)| p[3] == 255)); + assert!(frame.bgra[3] < 3, "the corner fades out"); + // The checker shows both of its colors somewhere. + let blueish = frame + .bgra + .chunks_exact(4) + .filter(|p| p[0] > 150 && p[2] < 90 && p[1] < 120) + .count(); + assert!(blueish > 500, "{blueish}"); + // A texture uploads once, however many frames draw it. + render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + } + #[test] fn a_room_renders_lit_with_glass_and_glow() { if !crate::available() { @@ -1239,4 +1749,48 @@ mod tests { assert!(drawn > total / 8, "geometry fills the frame: {drawn}"); assert!(drawn < total, "and the background stays clear"); } + + /// The mean column of the strongly red pixels. + fn red_center(frame: &Frame) -> f64 { + let (mut sum, mut count) = (0.0, 0usize); + for (i, p) in frame.bgra.chunks_exact(4).enumerate() { + if p[2] > 120 && p[2] as i32 - p[0] as i32 > 50 { + sum += (i % frame.width as usize) as f64; + count += 1; + } + } + assert!(count > 1000, "the red box shows: {count}"); + sum / count as f64 + } + + #[test] + fn translucent_surfaces_show_what_is_behind_them_in_place() { + if !crate::available() { + return; + } + let camera = Camera { + yaw: 0.6, + pitch: 0.3, + zoom: 1.0, + }; + let render = |shell: &str| { + let script = format!( + r#"return ui.scene {{ height = 300, draw = function(s) + s:box {{ at = {{ 0, 0.5, 0 }}, size = 1, color = "chart2" }} + {shell} + end }}"# + ); + render_scene(&scene(&script), camera, 720.0, 360.0, 1.0, &palette).unwrap() + }; + let bare = red_center(&render("")); + // A faded shell around the box (like a cutaway wall) must not shift + // it the way glass would. + let shelled = red_center(&render( + r#"s:box { at = { 0, 0.6, 0 }, size = 1.8, color = ui.color("muted", 0.3) }"#, + )); + assert!( + (bare - shelled).abs() < 6.0, + "box moved from {bare} to {shelled}" + ); + } } diff --git a/crates/visual-gpu/src/shader.rs b/crates/visual-gpu/src/shader.rs index 505d14002f..9a94ba9885 100644 --- a/crates/visual-gpu/src/shader.rs +++ b/crates/visual-gpu/src/shader.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use wgpu::util::DeviceExt; use zeron_visual::shader::{Blend, MAX_BUFFERS, ShaderUniforms}; -use crate::{FORMAT, Frame, gpu}; +use crate::{EDGE_FADE, FORMAT, Frame, gpu}; /// Passes render in float, so feedback buffers keep precision and values /// above 1 survive to the bloom. @@ -22,6 +22,8 @@ pub struct ShaderSpec<'a> { pub inputs: &'a [String], pub blend: Blend, pub bloom: f32, + /// Images sampled as `iTexture0`-`3`. + pub textures: &'a [zeron_visual::TextureData], } /// One shader's textures between frames: each buffer's previous and @@ -52,7 +54,8 @@ pub(crate) struct Pipelines { passes: Mutex, String>>>, } -const FULLSCREEN: &str = "@vertex fn main(@builtin(vertex_index) i: u32) -> @builtin(position) vec4 { +const FULLSCREEN: &str = + "@vertex fn main(@builtin(vertex_index) i: u32) -> @builtin(position) vec4 { let p = vec2(f32((i << 1u) & 2u), f32(i & 2u)); return vec4(p * 2.0 - 1.0, 0.0, 1.0); }"; @@ -102,16 +105,22 @@ fn luma(c: vec3) -> f32 { return dot(c, vec3(0.2126, 0.7152, 0.0722)); let c = textureSampleLevel(source, source_sampler, uv, 0.0); let g = textureSampleLevel(glow, source_sampler, uv, 0.0).rgb * post.x; let rgb = clamp(c.rgb + g, vec3(0.0), vec3(1.0)); + // An opaque shader is a deliberate box; the others fade at the edges. if (post.y > 1.5) { return vec4(rgb, 1.0); } + let fade = edge_fade(i.clip.xy, vec2(textureDimensions(source))); if (post.y > 0.5) { let a = max(rgb.r, max(rgb.g, rgb.b)); if (a <= 0.0001) { return vec4(0.0); } - return vec4(rgb / a, a); + return vec4(rgb / a, a * fade); } + // The shader's color is straight over its alpha; the glow is light over + // nothing. Add them premultiplied, then return straight alpha. let a = clamp(c.a + luma(g), 0.0, 1.0); - return vec4(rgb, a); + if (a <= 0.0001) { return vec4(0.0); } + let straight = clamp((clamp(c.rgb, vec3(0.0), vec3(1.0)) * c.a + g) / a, vec3(0.0), vec3(1.0)); + return vec4(straight, a * fade); } "#; @@ -154,6 +163,10 @@ impl Pipelines { texture_entry(5), sampler_entry(6), sampler_entry(7), + texture_entry(8), + texture_entry(9), + texture_entry(10), + texture_entry(11), ], }); let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { @@ -181,7 +194,7 @@ impl Pipelines { }); let post = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("shader post"), - source: wgpu::ShaderSource::Wgsl(POST.into()), + source: wgpu::ShaderSource::Wgsl(format!("{EDGE_FADE}{POST}").into()), }); let post_pipeline = |label: &str, entry: &str, format| { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { @@ -217,6 +230,8 @@ impl Pipelines { address_mode_v: address, mag_filter: wgpu::FilterMode::Linear, min_filter: wgpu::FilterMode::Linear, + // Image textures have mips; channels and noise have one level. + mipmap_filter: wgpu::MipmapFilterMode::Linear, ..Default::default() }) }; @@ -286,49 +301,53 @@ impl Pipelines { } /// The pipeline for one complete fragment program, compiled once. - fn pass(&self, device: &wgpu::Device, program: &str) -> Result, String> { + fn pass( + &self, + device: &wgpu::Device, + program: &str, + ) -> Result, String> { let mut passes = self.passes.lock().unwrap_or_else(|e| e.into_inner()); if let Some(cached) = passes.get(program) { return cached.clone(); } - let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); - let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("visual shader"), - source: wgpu::ShaderSource::Glsl { - shader: program.into(), - stage: wgpu::naga::ShaderStage::Fragment, - defines: &[], - }, - }); - let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("visual shader"), - layout: Some(&self.pipeline_layout), - vertex: wgpu::VertexState { - module: &self.fullscreen, - entry_point: Some("main"), - compilation_options: Default::default(), - buffers: &[], - }, - primitive: wgpu::PrimitiveState::default(), - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - fragment: Some(wgpu::FragmentState { - module: &module, - entry_point: Some("main"), - compilation_options: Default::default(), - targets: &[Some(wgpu::ColorTargetState { - format: HDR, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - }), - multiview_mask: None, - cache: None, + let compiled = crate::guarded(device, || { + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("visual shader"), + source: wgpu::ShaderSource::Glsl { + shader: program.into(), + stage: wgpu::naga::ShaderStage::Fragment, + defines: &[], + }, + }); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("visual shader"), + layout: Some(&self.pipeline_layout), + vertex: wgpu::VertexState { + module: &self.fullscreen, + entry_point: Some("main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: HDR, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) }); - let result = match pollster::block_on(scope.pop()) { - Some(error) => Err(format!("shader: {error}")), - None => Ok(Arc::new(pipeline)), - }; + let result = compiled + .map(Arc::new) + .map_err(|error| format!("shader: {error}")); // Keep the cache bounded; a session rarely holds more than a few. if passes.len() > 64 { passes.clear(); @@ -338,7 +357,12 @@ impl Pipelines { } } -fn texture(device: &wgpu::Device, label: &str, size: (u32, u32), format: wgpu::TextureFormat) -> wgpu::Texture { +fn texture( + device: &wgpu::Device, + label: &str, + size: (u32, u32), + format: wgpu::TextureFormat, +) -> wgpu::Texture { device.create_texture(&wgpu::TextureDescriptor { label: Some(label), size: wgpu::Extent3d { @@ -390,15 +414,28 @@ pub fn render_shader( height: u32, state: &mut Option, ) -> Result { - let gpu = gpu().ok_or("this device has no GPU for shaders")?; + let gpu = gpu().ok_or_else(|| { + format!( + "this device has no GPU for shaders ({})", + crate::unavailable_reason().unwrap_or_default() + ) + })?; let width = width.clamp(1, 8192); let height = height.clamp(1, 8192); let buffers = spec.buffers.len().min(MAX_BUFFERS); let device = &gpu.device; - let pipelines = gpu.shaders.get_or_init(|| Pipelines::new(device, &gpu.queue)); + let pipelines = gpu + .shaders + .get_or_init(|| Pipelines::new(device, &gpu.queue)); // Compile every pass first, so an error leaves the state untouched. let mut programs = Vec::with_capacity(buffers + 1); - for code in spec.buffers.iter().take(buffers).map(String::as_str).chain([spec.image]) { + for code in spec + .buffers + .iter() + .take(buffers) + .map(String::as_str) + .chain([spec.image]) + { let (program, _) = zeron_visual::shader::program(code, spec.inputs); programs.push(pipelines.pass(device, &program)?); } @@ -423,6 +460,18 @@ pub fn render_shader( let view = |t: &wgpu::Texture| t.create_view(&Default::default()); let empty = view(&pipelines.empty); let noise = view(&pipelines.noise); + let images: Vec = (0..zeron_visual::shader::MAX_TEXTURES) + .map(|i| { + spec.textures + .get(i) + .and_then(|data| { + gpu.textures + .get(device, &gpu.queue, data, crate::textures::Space::Raw) + .ok() + }) + .map_or_else(|| empty.clone(), |loaded| loaded.view) + }) + .collect(); // Channel `j` as pass `pass` sees it: an earlier buffer's frame from // this frame, its own and later ones' from the last. let channel = |pass: usize, j: usize| -> wgpu::TextureView { @@ -471,6 +520,22 @@ pub fn render_shader( binding: 7, resource: wgpu::BindingResource::Sampler(&pipelines.repeat), }, + wgpu::BindGroupEntry { + binding: 8, + resource: wgpu::BindingResource::TextureView(&images[0]), + }, + wgpu::BindGroupEntry { + binding: 9, + resource: wgpu::BindingResource::TextureView(&images[1]), + }, + wgpu::BindGroupEntry { + binding: 10, + resource: wgpu::BindingResource::TextureView(&images[2]), + }, + wgpu::BindGroupEntry { + binding: 11, + resource: wgpu::BindingResource::TextureView(&images[3]), + }, ], }); let target = if pass < buffers { @@ -618,12 +683,18 @@ mod tests { inputs: &[], blend, bloom: 0.0, + textures: &[], } } fn px(frame: &Frame, x: u32, y: u32) -> [u8; 4] { let o = ((y * frame.width + x) * 4) as usize; - [frame.bgra[o + 2], frame.bgra[o + 1], frame.bgra[o], frame.bgra[o + 3]] + [ + frame.bgra[o + 2], + frame.bgra[o + 1], + frame.bgra[o], + frame.bgra[o + 3], + ] } #[test] @@ -640,13 +711,83 @@ mod tests { ..Default::default() }; uniforms.time[0] = 0.5; - let frame = - render_shader(&spec(code, &[], Blend::Opaque), &uniforms, 64, 32, &mut None).unwrap(); + let frame = render_shader( + &spec(code, &[], Blend::Opaque), + &uniforms, + 64, + 32, + &mut None, + ) + .unwrap(); // Red grows to the right; green grows upward (Shadertoy's origin is // bottom-left), so the top row is the greenest. assert!(px(&frame, 60, 16)[0] > px(&frame, 4, 16)[0]); assert!(px(&frame, 32, 2)[1] > px(&frame, 32, 29)[1]); - assert!((i32::from(px(&frame, 32, 16)[2]) - 128).abs() < 3, "iTime reaches the shader"); + assert!( + (i32::from(px(&frame, 32, 16)[2]) - 128).abs() < 3, + "iTime reaches the shader" + ); + } + + #[test] + fn image_textures_are_sampled_as_stored() { + if !crate::available() { + return; + } + let code = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + fragColor = texture(iTexture0, fragCoord / iResolution.xy);\n\ + }"; + // Left texel red, right texel a mid blue (stored values, not sRGB). + let textures = [zeron_visual::TextureData::Rgba { + width: 2, + height: 1, + pixels: vec![255, 0, 0, 255, 0, 0, 128, 255].into(), + }]; + let spec = ShaderSpec { + textures: &textures, + ..spec(code, &[], Blend::Opaque) + }; + let uniforms = ShaderUniforms { + resolution: [64.0, 16.0, 1.0, 0.0], + ..Default::default() + }; + let frame = render_shader(&spec, &uniforms, 64, 16, &mut None).unwrap(); + // At the texel centers (it repeats, so edges blend with the far side). + let left = px(&frame, 16, 8); + assert!(left[0] > 245 && left[2] < 8, "{left:?}"); + let right = px(&frame, 48, 8); + assert!( + right[0] < 8 && (i32::from(right[2]) - 128).abs() < 6, + "{right:?}" + ); + } + + #[test] + fn image_textures_show_upright_as_on_shadertoy() { + if !crate::available() { + return; + } + let code = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + fragColor = texture(iTexture0, fragCoord / iResolution.xy);\n\ + }"; + // Top texel red, bottom texel blue, as an image stores them. + let textures = [zeron_visual::TextureData::Rgba { + width: 1, + height: 2, + pixels: vec![255, 0, 0, 255, 0, 0, 255, 255].into(), + }]; + let spec = ShaderSpec { + textures: &textures, + ..spec(code, &[], Blend::Opaque) + }; + let uniforms = ShaderUniforms { + resolution: [16.0, 64.0, 1.0, 0.0], + ..Default::default() + }; + let frame = render_shader(&spec, &uniforms, 16, 64, &mut None).unwrap(); + let (top, bottom) = (px(&frame, 8, 16), px(&frame, 8, 48)); + assert!(top[0] > 245 && top[2] < 8, "{top:?}"); + assert!(bottom[2] > 245 && bottom[0] < 8, "{bottom:?}"); } #[test] @@ -669,6 +810,34 @@ mod tests { assert_eq!(px(&frame, 24, 24)[3], 255, "light is solid"); } + #[test] + fn light_fades_out_toward_the_edges_unless_opaque() { + if !crate::available() { + return; + } + // White everywhere, like a beam running off the frame. + let code = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + fragColor = vec4(1.0);\n\ + }"; + let uniforms = ShaderUniforms { + resolution: [200.0, 100.0, 1.0, 0.0], + ..Default::default() + }; + let render = + |blend| render_shader(&spec(code, &[], blend), &uniforms, 200, 100, &mut None).unwrap(); + let light = render(Blend::Light); + // The ramp is 15 px (15% of 100): clear at the edge, solid inside, + // rising steadily in between. + let row: Vec = (0..20).map(|x| px(&light, x, 50)[3]).collect(); + assert!(row[0] < 3, "{row:?}"); + assert_eq!(row[16], 255, "{row:?}"); + assert!(row.windows(2).all(|w| w[0] <= w[1]), "{row:?}"); + assert!(px(&light, 100, 0)[3] < 3, "the top fades too"); + assert!(px(&light, 199, 99)[3] < 3, "and the corners"); + let opaque = render(Blend::Opaque); + assert_eq!(px(&opaque, 0, 0)[3], 255, "an opaque shader keeps its box"); + } + #[test] fn a_buffer_reads_its_own_previous_frame() { if !crate::available() { @@ -676,11 +845,13 @@ mod tests { } // Buffer A counts frames by adding 0.25 to what it wrote last time; // the image shows it. - let buffer = vec!["void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ + let buffer = vec![ + "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ vec4 last = texture(iChannel0, fragCoord / iResolution.xy);\n\ fragColor = vec4(last.r + 0.25, 0.0, 0.0, 1.0);\n\ }" - .to_owned()]; + .to_owned(), + ]; let image = "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n\ fragColor = vec4(texture(iChannel0, fragCoord / iResolution.xy).r, 0.0, 0.0, 1.0);\n\ }"; @@ -691,12 +862,24 @@ mod tests { let mut state = None; let mut reds = Vec::new(); for _ in 0..3 { - let frame = render_shader(&spec(image, &buffer, Blend::Opaque), &uniforms, 16, 16, &mut state) - .unwrap(); + let frame = render_shader( + &spec(image, &buffer, Blend::Opaque), + &uniforms, + 16, + 16, + &mut state, + ) + .unwrap(); reds.push(px(&frame, 8, 8)[0]); } - assert!(reds[0] < reds[1] && reds[1] < reds[2], "accumulates: {reds:?}"); - assert!((i32::from(reds[2]) - 191).abs() < 4, "0.75 after three frames: {reds:?}"); + assert!( + reds[0] < reds[1] && reds[1] < reds[2], + "accumulates: {reds:?}" + ); + assert!( + (i32::from(reds[2]) - 191).abs() < 4, + "0.75 after three frames: {reds:?}" + ); } #[test] diff --git a/crates/visual-gpu/src/textures.rs b/crates/visual-gpu/src/textures.rs new file mode 100644 index 0000000000..bf07718c9c --- /dev/null +++ b/crates/visual-gpu/src/textures.rs @@ -0,0 +1,386 @@ +//! Image textures for scenes and shaders: decoded, mipmapped, and uploaded +//! once per asset, then shared by every frame and every visual that uses +//! the same bytes. + +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex, Weak}; + +use zeron_visual::TextureData; +use zeron_visual::image::{Image, Pixels}; + +/// What the cache may hold on the GPU before it drops the least recently +/// used. +const BUDGET_BYTES: usize = 256 * 1024 * 1024; + +/// How a texture's 8-bit values are read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum Space { + /// Colors as images store them (sRGB), sampled as linear light. + Color, + /// Values as stored, as Shadertoy samples them (bottom row first). + Raw, +} + +/// A texture ready to bind. +#[derive(Clone)] +pub(crate) struct Loaded { + pub(crate) view: wgpu::TextureView, + pub(crate) mips: u32, +} + +/// The bytes' content and how they're read: the same image uploads once, +/// whichever visual (or re-parse of one) brings it. +type Key = (u64, usize, Space); + +struct Entry { + key: Key, + loaded: Result, + /// GPU bytes (failures count a little, so they're evicted too). + size: usize, + used: u64, +} + +/// An `Arc` already hashed, while it lives: hashing once per allocation, not +/// per frame. +struct Alias { + bytes: Weak<[u8]>, + hash: u64, +} + +#[derive(Default)] +struct State { + entries: Vec, + aliases: Vec, + clock: u64, +} + +#[derive(Default)] +pub(crate) struct Textures { + state: Mutex, +} + +fn bytes_of(data: &TextureData) -> &Arc<[u8]> { + match data { + TextureData::Encoded(bytes) => bytes, + TextureData::Rgba { pixels, .. } => pixels, + } +} + +fn content_hash(bytes: &[u8]) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut hasher); + hasher.finish() +} + +impl Textures { + /// `data` on the GPU, decoding and uploading it on first use. Decoding + /// runs outside the lock, so a large image doesn't hold up other renders. + pub(crate) fn get( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + data: &TextureData, + space: Space, + ) -> Result { + let bytes = bytes_of(data); + let known = { + let mut state = self.state.lock().unwrap_or_else(|e| e.into_inner()); + state.aliases.retain(|a| a.bytes.strong_count() > 0); + state + .aliases + .iter() + .find(|a| std::ptr::addr_eq(a.bytes.as_ptr(), Arc::as_ptr(bytes))) + .map(|a| a.hash) + }; + let hash = known.unwrap_or_else(|| content_hash(bytes)); + let key = (hash, bytes.len(), space); + { + let mut guard = self.state.lock().unwrap_or_else(|e| e.into_inner()); + let state = &mut *guard; + if known.is_none() { + state.aliases.push(Alias { + bytes: Arc::downgrade(bytes), + hash, + }); + } + state.clock += 1; + if let Some(entry) = state.entries.iter_mut().find(|e| e.key == key) { + entry.used = state.clock; + return entry.loaded.clone(); + } + } + let loaded = zeron_visual::image::decode_texture(data) + .and_then(|image| upload(device, queue, &image, space)); + if let Err(error) = &loaded { + tracing::warn!(%error, "visual texture unusable"); + } + let size = loaded.as_ref().map_or(1024, |(_, size)| *size); + let loaded = loaded.map(|(loaded, _)| loaded); + let mut guard = self.state.lock().unwrap_or_else(|e| e.into_inner()); + let state = &mut *guard; + state.clock += 1; + if let Some(entry) = state.entries.iter_mut().find(|e| e.key == key) { + // Another render uploaded it meanwhile. + entry.used = state.clock; + return entry.loaded.clone(); + } + state.entries.push(Entry { + key, + loaded: loaded.clone(), + size, + used: state.clock, + }); + while state.entries.iter().map(|e| e.size).sum::() > BUDGET_BYTES + && state.entries.len() > 1 + { + let oldest = state + .entries + .iter() + .enumerate() + .min_by_key(|(_, e)| e.used) + .map(|(i, _)| i) + .expect("not empty"); + state.entries.swap_remove(oldest); + } + loaded + } +} + +/// An `f32` as IEEE half-precision bits (finite light values only). +fn half(value: f32) -> u16 { + let value = if value.is_finite() { + value.clamp(0.0, 65504.0) + } else { + 0.0 + }; + let bits = value.to_bits(); + let exponent = ((bits >> 23) & 0xff) as i32 - 127 + 15; + let mantissa = bits & 0x7f_ffff; + if exponent <= 0 { + // Too small for a normal half: flush to zero. + 0 + } else { + ((exponent as u16) << 10) | (mantissa >> 13) as u16 + } +} + +/// Every mip level of `image`, largest first, as the texture's bytes (one +/// buffer), with the level count and format. +fn levels(image: &Image) -> (Vec, u32, wgpu::TextureFormat) { + let (mut w, mut h) = (image.width as usize, image.height as usize); + let mut mips = 1; + match &image.pixels { + Pixels::Rgba8(pixels) => { + let mut data = pixels.clone(); + let mut level_start = 0; + while w > 1 || h > 1 { + let (next, nw, nh) = downsample(&data[level_start..], w, h); + level_start = data.len(); + data.extend_from_slice(&next); + (w, h) = (nw, nh); + mips += 1; + } + (data, mips, wgpu::TextureFormat::Rgba8Unorm) + } + Pixels::RgbaF32(pixels) => { + let mut data: Vec = pixels.iter().flat_map(|v| half(*v).to_le_bytes()).collect(); + let mut level = None::>; + while w > 1 || h > 1 { + let (next, nw, nh) = downsample(level.as_deref().unwrap_or(pixels), w, h); + data.extend(next.iter().flat_map(|v| half(*v).to_le_bytes())); + level = Some(next); + (w, h) = (nw, nh); + mips += 1; + } + (data, mips, wgpu::TextureFormat::Rgba16Float) + } + } +} + +/// Half the size, each texel the mean of the 2x2 it covers. +fn downsample + FromF32>( + src: &[T], + w: usize, + h: usize, +) -> (Vec, usize, usize) { + let (nw, nh) = ((w / 2).max(1), (h / 2).max(1)); + let mut out = Vec::with_capacity(nw * nh * 4); + for y in 0..nh { + for x in 0..nw { + for c in 0..4 { + let mut sum = 0.0f32; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let sx = (x * 2 + dx).min(w - 1); + let sy = (y * 2 + dy).min(h - 1); + sum += src[(sy * w + sx) * 4 + c].into(); + } + out.push(T::from_f32(sum / 4.0)); + } + } + } + (out, nw, nh) +} + +trait FromF32 { + fn from_f32(value: f32) -> Self; +} +impl FromF32 for u8 { + fn from_f32(value: f32) -> Self { + value.round().clamp(0.0, 255.0) as u8 + } +} +impl FromF32 for f32 { + fn from_f32(value: f32) -> Self { + value + } +} + +fn upload( + device: &wgpu::Device, + queue: &wgpu::Queue, + image: &Image, + space: Space, +) -> Result<(Loaded, usize), String> { + use wgpu::util::DeviceExt; + let largest = device.limits().max_texture_dimension_2d; + if image.width.max(image.height) > largest { + return Err(format!( + "a {}x{} image is larger than this GPU's {largest}px textures", + image.width, image.height + )); + } + // Shadertoy samples images bottom row first, as its passes store rows. + let flipped; + let image = if space == Space::Raw { + flipped = flip_rows(image); + &flipped + } else { + image + }; + let (data, mips, format) = levels(image); + let format = match (format, space) { + (wgpu::TextureFormat::Rgba8Unorm, Space::Color) => wgpu::TextureFormat::Rgba8UnormSrgb, + (format, _) => format, + }; + let size = data.len(); + let texture = crate::guarded(device, || { + device.create_texture_with_data( + queue, + &wgpu::TextureDescriptor { + label: Some("visual texture"), + size: wgpu::Extent3d { + width: image.width, + height: image.height, + depth_or_array_layers: 1, + }, + mip_level_count: mips, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }, + wgpu::util::TextureDataOrder::LayerMajor, + &data, + ) + })?; + Ok(( + Loaded { + view: texture.create_view(&Default::default()), + mips, + }, + size, + )) +} + +/// The image upside down. +fn flip_rows(image: &Image) -> Image { + fn flip(pixels: &[T], width: usize) -> Vec { + pixels + .chunks_exact(width * 4) + .rev() + .flatten() + .copied() + .collect() + } + let width = image.width as usize; + Image { + width: image.width, + height: image.height, + pixels: match &image.pixels { + Pixels::Rgba8(p) => Pixels::Rgba8(flip(p, width)), + Pixels::RgbaF32(p) => Pixels::RgbaF32(flip(p, width)), + }, + } +} + +/// A 1x1 texture of one color, for slots with nothing to sample. +pub(crate) fn solid( + device: &wgpu::Device, + queue: &wgpu::Queue, + rgba: [u8; 4], +) -> wgpu::TextureView { + use wgpu::util::DeviceExt; + device + .create_texture_with_data( + queue, + &wgpu::TextureDescriptor { + label: Some("visual solid texture"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }, + wgpu::util::TextureDataOrder::LayerMajor, + &rgba, + ) + .create_view(&Default::default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn halves_round_trip_light_values() { + assert_eq!(half(0.0), 0); + assert_eq!(half(1.0), 0x3c00); + assert_eq!(half(2.0), 0x4000); + assert_eq!(half(0.5), 0x3800); + assert_eq!(half(f32::INFINITY), 0); + assert_eq!(half(1e9), 0x7bff); + } + + #[test] + fn mip_chains_reach_one_texel() { + let image = Image { + width: 5, + height: 2, + pixels: Pixels::Rgba8(vec![200; 5 * 2 * 4]), + }; + let (data, mips, _) = levels(&image); + assert_eq!(mips, 3); + assert_eq!(data.len(), 5 * 2 * 4 + 2 * 4 + 4); + assert_eq!(&data[data.len() - 4..], [200; 4]); + } + + #[test] + fn raw_images_flip_to_bottom_row_first() { + let image = Image { + width: 1, + height: 2, + pixels: Pixels::Rgba8(vec![1, 1, 1, 1, 2, 2, 2, 2]), + }; + let Pixels::Rgba8(pixels) = flip_rows(&image).pixels else { + unreachable!() + }; + assert_eq!(pixels, [2, 2, 2, 2, 1, 1, 1, 1]); + } +} diff --git a/crates/visual/Cargo.toml b/crates/visual/Cargo.toml index 8d5bb82e81..1b19b58662 100644 --- a/crates/visual/Cargo.toml +++ b/crates/visual/Cargo.toml @@ -10,4 +10,10 @@ description = "Sandboxed Luau runtime for agent visuals: scripts build a native mlua = { version = "0.12", features = ["luau"] } # The GLSL compiler wgpu uses, so a shader is validated exactly as it will run. naga = { version = "29", features = ["glsl-in"] } +# glTF models (`s:model`): `.glb` or self-contained `.gltf`. +gltf = { version = "1", default-features = false, features = ["import", "utils", "names", "KHR_materials_transmission", "KHR_materials_ior", "KHR_materials_emissive_strength"] } +# Texture and environment-map assets. +image = { version = "0.25.10", default-features = false, features = ["png", "jpeg", "hdr", "webp"] } thiserror.workspace = true +# Images embedded in glTF models as data URIs. +base64 = "0.22" diff --git a/crates/visual/src/assets.rs b/crates/visual/src/assets.rs new file mode 100644 index 0000000000..609cbc0575 --- /dev/null +++ b/crates/visual/src/assets.rs @@ -0,0 +1,82 @@ +//! Files an agent ships with a visual (images, glTF models, environment +//! maps), by name. The agent passes them to `show_visual`; the engine stores +//! them beside the script, and every runtime that loads the script gets them. +//! Scripts refer to them by name (`texture = "wood.jpg"`, +//! `s:model { asset = "prism.glb" }`). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +/// At most this many assets per visual. +pub const MAX_ASSETS: usize = 16; +/// Largest single asset. +pub const MAX_ASSET_BYTES: usize = 8 * 1024 * 1024; +/// All of a visual's assets together. +pub const MAX_TOTAL_ASSET_BYTES: usize = 10 * 1024 * 1024; + +/// A visual's named files, cheap to clone. +#[derive(Clone, Default)] +pub struct Assets { + files: Arc>>, + /// Parsed models, so a scene redrawing every frame parses each once. + pub(crate) models: Arc>>>, +} + +impl std::fmt::Debug for Assets { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.files.keys()).finish() + } +} + +impl PartialEq for Assets { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.files, &other.files) + } +} + +impl Assets { + pub fn new(files: HashMap>) -> Self { + Self { + files: Arc::new(files), + models: Arc::default(), + } + } + + pub fn get(&self, name: &str) -> Option> { + self.files.get(name).cloned() + } + + pub fn names(&self) -> impl Iterator { + self.files.keys().map(String::as_str) + } + + pub fn is_empty(&self) -> bool { + self.files.is_empty() + } + + /// The file `name`, or an error naming what the visual does have. + pub(crate) fn require(&self, name: &str) -> Result, String> { + self.get(name).ok_or_else(|| { + let mut known: Vec<&str> = self.names().collect(); + known.sort_unstable(); + if known.is_empty() { + format!("no asset named '{name}' (pass files in show_visual's assets)") + } else { + format!( + "no asset named '{name}' (this visual has {})", + known.join(", ") + ) + } + }) + } +} + +/// Whether `name` can name an asset: a plain file name. +pub fn valid_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 64 + && !name.starts_with('.') + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) +} diff --git a/crates/visual/src/draw.rs b/crates/visual/src/draw.rs index f08402b37f..3d38d2e9a5 100644 --- a/crates/visual/src/draw.rs +++ b/crates/visual/src/draw.rs @@ -57,6 +57,9 @@ pub struct Graphics { pub time: f64, pub pointer: Option, pub commands: Vec, + /// The draw function looked at `pointer`: a touch host lets such a + /// canvas take drags instead of scrolling past it. + pub read_pointer: std::cell::Cell, points: usize, } @@ -68,6 +71,7 @@ impl Graphics { time, pointer, commands: Vec::new(), + read_pointer: std::cell::Cell::new(false), points: 0, } } @@ -244,15 +248,18 @@ impl UserData for Graphics { fields.add_field_method_get("width", |_, g| Ok(g.width)); fields.add_field_method_get("height", |_, g| Ok(g.height)); fields.add_field_method_get("time", |_, g| Ok(g.time)); - fields.add_field_method_get("pointer", |lua, g| match g.pointer { - Some(p) => { - let t = lua.create_table()?; - t.set("x", p.x)?; - t.set("y", p.y)?; - t.set("down", p.down)?; - Ok(Value::Table(t)) + fields.add_field_method_get("pointer", |lua, g| { + g.read_pointer.set(true); + match g.pointer { + Some(p) => { + let t = lua.create_table()?; + t.set("x", p.x)?; + t.set("y", p.y)?; + t.set("down", p.down)?; + Ok(Value::Table(t)) + } + None => Ok(Value::Nil), } - None => Ok(Value::Nil), }); } diff --git a/crates/visual/src/image.rs b/crates/visual/src/image.rs new file mode 100644 index 0000000000..6b5ce19e93 --- /dev/null +++ b/crates/visual/src/image.rs @@ -0,0 +1,110 @@ +//! Decoding image assets for textures and environment maps. + +use crate::scene::TextureData; + +/// Largest side an image asset may have. +pub const MAX_IMAGE_SIDE: u32 = 8192; +/// Most pixels an image may decode to (64 MB as RGBA8). +pub const MAX_IMAGE_PIXELS: u64 = 4096 * 4096; +/// Most pixels for HDR, which decodes to four floats a pixel (128 MB). +pub const MAX_HDR_PIXELS: u64 = 4096 * 2048; + +/// A decoded image, top row first. +#[derive(Debug, Clone)] +pub struct Image { + pub width: u32, + pub height: u32, + pub pixels: Pixels, +} + +#[derive(Debug, Clone)] +pub enum Pixels { + /// sRGB-encoded RGBA, straight alpha. + Rgba8(Vec), + /// Linear RGBA light (HDR environment maps). + RgbaF32(Vec), +} + +fn reader(bytes: &[u8]) -> Result>, String> { + image::ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format() + .map_err(|e| format!("unreadable image: {e}")) +} + +/// The image's size, without decoding it: a cheap validity check. +pub fn dimensions(bytes: &[u8]) -> Result<(u32, u32), String> { + let reader = reader(bytes)?; + if reader.format().is_none() { + return Err("not a PNG, JPEG, WebP, or Radiance HDR image".into()); + } + let hdr = reader.format() == Some(image::ImageFormat::Hdr); + let (width, height) = reader + .into_dimensions() + .map_err(|e| format!("unreadable image: {e}"))?; + check_size(width, height, hdr)?; + Ok((width, height)) +} + +/// Sides within 1..=MAX_IMAGE_SIDE and pixels within the decode budget. +fn check_size(width: u32, height: u32, hdr: bool) -> Result<(), String> { + if width == 0 || height == 0 || width.max(height) > MAX_IMAGE_SIDE { + return Err(format!( + "a {width}x{height} image; sides run 1 to {MAX_IMAGE_SIDE} pixels" + )); + } + let budget = if hdr { + MAX_HDR_PIXELS + } else { + MAX_IMAGE_PIXELS + }; + if u64::from(width) * u64::from(height) > budget { + return Err(format!( + "a {width}x{height} image; at most {budget} pixels{}", + if hdr { " for HDR" } else { "" } + )); + } + Ok(()) +} + +/// Decode an encoded image asset. +pub fn decode(bytes: &[u8]) -> Result { + dimensions(bytes)?; + let reader = reader(bytes)?; + let hdr = reader.format() == Some(image::ImageFormat::Hdr); + let image = reader + .decode() + .map_err(|e| format!("unreadable image: {e}"))?; + let (width, height) = (image.width(), image.height()); + let pixels = if hdr { + Pixels::RgbaF32(image.into_rgba32f().into_raw()) + } else { + Pixels::Rgba8(image.into_rgba8().into_raw()) + }; + Ok(Image { + width, + height, + pixels, + }) +} + +/// Decode any texture a scene uses. +pub fn decode_texture(data: &TextureData) -> Result { + match data { + TextureData::Encoded(bytes) => decode(bytes), + TextureData::Rgba { + width, + height, + pixels, + } => { + check_size(*width, *height, false)?; + if pixels.len() != *width as usize * *height as usize * 4 { + return Err("image pixels don't match its size".into()); + } + Ok(Image { + width: *width, + height: *height, + pixels: Pixels::Rgba8(pixels.to_vec()), + }) + } + } +} diff --git a/crates/visual/src/lib.rs b/crates/visual/src/lib.rs index 97b6c0c187..1a4ac13f89 100644 --- a/crates/visual/src/lib.rs +++ b/crates/visual/src/lib.rs @@ -17,7 +17,10 @@ //! app. `math.random` is seeded from the visual's id before every call, so a //! script's random data is the same on every render and every device. +pub mod assets; mod draw; +pub mod image; +mod model; mod node; mod raster; mod scene; @@ -30,14 +33,17 @@ use std::time::{Duration, Instant}; use mlua::{Function, Lua, LuaOptions, StdLib, Table, Value, VmState}; +pub use assets::Assets; pub use draw::{DrawCmd, DrawText, Pointer}; pub use node::{ Axis, Canvas, Chart, ChartKind, Column, HandlerId, Node, NodeKind, NumberFormat, SceneNode, - ShaderNode, - Series, Span, Stack, Table_ as TableNode, Text, + Series, ShaderNode, Span, Stack, Table_ as TableNode, Text, }; pub use raster::{Quality, Raster, RasterLabel}; -pub use scene::{Camera, CameraFrame, FaceView, Material, Prim, Projection, Scene, V3}; +pub use scene::{ + Camera, CameraFrame, Environment, FaceView, Material, Prim, Projection, Scene, Sprite, + TextureData, TextureSource, V3, +}; pub use style::{Align, CHART_SERIES, Color, Justify, TOKEN_NAMES, TextSize, Token, Weight}; /// Largest script `show_visual` accepts. @@ -105,6 +111,14 @@ pub enum HandlerArg { Point(f32, f32), } +/// One run of a canvas's draw function. +#[derive(Debug, Clone, PartialEq)] +pub struct CanvasDrawing { + pub commands: Vec, + /// The drawing read `g.pointer`, so it responds to where the pointer is. + pub reads_pointer: bool, +} + /// One visual's live program. pub struct Visual { lua: Lua, @@ -114,6 +128,7 @@ pub struct Visual { seed: i64, deadline: Rc>>, logs: Rc>>, + assets: Assets, } const MAX_LOG_LINES: usize = 200; @@ -232,9 +247,20 @@ impl Visual { seed, deadline, logs, + assets: Assets::default(), }) } + /// Give the script its asset files (textures, models). + pub fn with_assets(mut self, assets: Assets) -> Self { + self.assets = assets; + self + } + + pub fn assets(&self) -> &Assets { + &self.assets + } + fn budgeted( &self, budget: Duration, @@ -288,6 +314,21 @@ impl Visual { time: f64, pointer: Option, ) -> Result, VisualError> { + Ok(self + .draw_canvas_traced(id, width, height, time, pointer)? + .commands) + } + + /// [`Visual::draw_canvas`], also reporting whether the drawing read the + /// pointer. + pub fn draw_canvas_traced( + &mut self, + id: HandlerId, + width: f32, + height: f32, + time: f64, + pointer: Option, + ) -> Result { let draw = self.handler(id)?.clone(); let graphics = self .lua @@ -295,13 +336,19 @@ impl Visual { self.budgeted(DRAW_BUDGET, || { draw.call::<()>((&graphics, width, height, time)) })?; - Ok(graphics.take::()?.commands) + let graphics = graphics.take::()?; + Ok(CanvasDrawing { + reads_pointer: graphics.read_pointer.get(), + commands: graphics.commands, + }) } /// Run a scene's draw function. pub fn draw_scene(&mut self, id: HandlerId, time: f64) -> Result { let draw = self.handler(id)?.clone(); - let scene = self.lua.create_userdata(Scene::new(time))?; + let scene = self + .lua + .create_userdata(Scene::with_assets(time, self.assets.clone()))?; self.budgeted(DRAW_BUDGET, || draw.call::<()>((&scene, time)))?; Ok(scene.take::()?) } @@ -358,7 +405,12 @@ pub struct Report { /// once with a render after each. The first failure comes back with where /// it happened, so the agent can fix the script before the reader sees it. pub fn check(source: &str, seed: u64) -> Result { - let mut visual = Visual::load(source, seed)?; + check_with_assets(source, seed, Assets::default()) +} + +/// [`check`] for a script shipped with asset files. +pub fn check_with_assets(source: &str, seed: u64, assets: Assets) -> Result { + let mut visual = Visual::load(source, seed)?.with_assets(assets); let tree = visual.render()?; let mut report = Report::default(); exercise_draws(&mut visual, &tree, &mut report)?; @@ -425,21 +477,40 @@ fn exercise_draws( ) .prims .is_empty() + && drawn.sprites().is_empty() { return Err(VisualError::script( "a scene draw added no geometry (use s:box, s:sphere, s:surface, s:mesh, \ - s:line, s:point)", + s:line, s:point, s:model, s:sprite)", )); } + for texture in drawn.textures() { + if let TextureData::Encoded(bytes) = &texture.data { + crate::image::dimensions(bytes).map_err(|e| { + VisualError::script(format!( + "in a scene draw: {}: {e}", + texture.key.trim_start_matches("asset:") + )) + })?; + } + } } NodeKind::Shader(shader) => { let names: Vec = shader.inputs.iter().map(|(n, _)| n.clone()).collect(); for (i, buffer) in shader.buffers.iter().enumerate() { - crate::shader::validate(buffer, &names).map_err(|e| { - VisualError::script(format!("in shader buffer {}: {e}", i + 1)) - })?; + crate::shader::validate(buffer, &names) + .map_err(|e| VisualError::script(format!("in shader buffer {}: {e}", i + 1)))?; } crate::shader::validate(&shader.code, &names).map_err(VisualError::script)?; + for name in &shader.textures { + visual + .assets + .require(name) + .and_then(|bytes| { + crate::image::dimensions(&bytes).map_err(|e| format!("'{name}': {e}")) + }) + .map_err(|e| VisualError::script(format!("shader textures: {e}")))?; + } } _ => {} } diff --git a/crates/visual/src/model.rs b/crates/visual/src/model.rs new file mode 100644 index 0000000000..b40ed9b180 --- /dev/null +++ b/crates/visual/src/model.rs @@ -0,0 +1,458 @@ +//! glTF models for `s:model`: the default scene's meshes flattened to +//! world-space triangles with their normals, texture coordinates, and PBR +//! materials, plus the images those materials use. + +use std::sync::Arc; + +use crate::scene::{Material, TextureData, V3}; + +/// A parsed model, ready to place into a scene. +#[derive(Debug)] +pub struct Model { + pub(crate) triangles: Vec, + pub(crate) materials: Vec, + /// Encoded images (decoded, size-checked, only when drawn), indexed by + /// `ModelMaterial::texture`. + pub(crate) images: Vec, + pub(crate) bounds: (V3, V3), +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Triangle { + pub(crate) positions: [V3; 3], + pub(crate) normals: Option<[V3; 3]>, + pub(crate) uvs: Option<[[f32; 2]; 3]>, + pub(crate) material: usize, +} + +#[derive(Debug, Clone)] +pub(crate) struct ModelMaterial { + /// sRGB `0xRRGGBB` and alpha. + pub(crate) rgb: u32, + pub(crate) alpha: f32, + pub(crate) texture: Option, + pub(crate) material: Material, + pub(crate) double_sided: bool, +} + +/// Most triangles one model may bring. +const MAX_TRIANGLES: usize = 400_000; +/// Most node visits while walking the scene (instancing can repeat nodes; +/// a malformed file can loop). +const MAX_NODE_VISITS: usize = 100_000; + +fn to_srgb(c: f32) -> f32 { + let c = c.clamp(0.0, 1.0); + if c <= 0.0031308 { + c * 12.92 + } else { + 1.055 * c.powf(1.0 / 2.4) - 0.055 + } +} + +fn rgb_of(linear: [f32; 3]) -> u32 { + let channel = |c: f32| (to_srgb(c) * 255.0).round() as u32; + channel(linear[0]) << 16 | channel(linear[1]) << 8 | channel(linear[2]) +} + +type Mat4 = [[f32; 4]; 4]; + +fn multiply(a: &Mat4, b: &Mat4) -> Mat4 { + // Column-major, as glTF stores them: (a * b)[col][row]. + let mut out = [[0.0; 4]; 4]; + for (col, out_col) in out.iter_mut().enumerate() { + for (row, value) in out_col.iter_mut().enumerate() { + *value = (0..4).map(|k| a[k][row] * b[col][k]).sum(); + } + } + out +} + +fn transform_point(m: &Mat4, p: [f32; 3]) -> V3 { + [ + m[0][0] * p[0] + m[1][0] * p[1] + m[2][0] * p[2] + m[3][0], + m[0][1] * p[0] + m[1][1] * p[1] + m[2][1] * p[2] + m[3][1], + m[0][2] * p[0] + m[1][2] * p[1] + m[2][2] * p[2] + m[3][2], + ] +} + +/// Normals transform by the matrix's linear part; uniform scales are by far +/// the common case, and the shading normal is renormalized anyway. +fn transform_normal(m: &Mat4, n: [f32; 3]) -> V3 { + let v = [ + m[0][0] * n[0] + m[1][0] * n[1] + m[2][0] * n[2], + m[0][1] * n[0] + m[1][1] * n[1] + m[2][1] * n[2], + m[0][2] * n[0] + m[1][2] * n[1] + m[2][2] * n[2], + ]; + let len = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2]).sqrt().max(1e-9); + [v[0] / len, v[1] / len, v[2] / len] +} + +impl Model { + /// Parse a `.glb` or self-contained `.gltf` (buffers and images embedded + /// or as data URIs; no external files). + pub fn parse(bytes: &[u8]) -> Result { + // The glTF reader asserts on some malformed files rather than + // erroring; a model is untrusted, so that's an error here too. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::parse_unguarded(bytes) + })) + .unwrap_or_else(|_| Err("not a usable glTF model: malformed accessors".into())) + } + + fn parse_unguarded(bytes: &[u8]) -> Result { + let unusable = |e: gltf::Error| format!("not a usable glTF model: {e}"); + let gltf = gltf::Gltf::from_slice(bytes).map_err(unusable)?; + let document = gltf.document; + // No base path: external files are refused, only embedded data loads. + let buffers = gltf::import_buffers(&document, None, gltf.blob).map_err(unusable)?; + // Images stay encoded: the renderer decodes them with the same size + // limits as image assets (a small file can declare a huge image). + let images = document + .images() + .map(|image| { + let bytes: Vec = match image.source() { + gltf::image::Source::View { view, .. } => buffers + .get(view.buffer().index()) + .and_then(|data| { + data.0 + .get(view.offset()..view.offset().checked_add(view.length())?) + }) + .map(<[u8]>::to_vec) + .unwrap_or_default(), + gltf::image::Source::Uri { uri, .. } => uri + .strip_prefix("data:") + .and_then(|rest| rest.split_once(";base64,")) + .and_then(|(_, data)| { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.decode(data).ok() + }) + .unwrap_or_default(), + }; + TextureData::Encoded(bytes.into()) + }) + .collect(); + let materials: Vec = document + .materials() + .map(|m| { + let pbr = m.pbr_metallic_roughness(); + let base = pbr.base_color_factor(); + let emissive = m.emissive_factor(); + let strength = m.emissive_strength().unwrap_or(1.0); + let glow = emissive.iter().copied().fold(0.0f32, f32::max) * strength; + let mut rgb = rgb_of([base[0], base[1], base[2]]); + // An emitter whose base is dark shines its emissive color. + if glow > 0.0 && base[0].max(base[1]).max(base[2]) < 0.05 { + rgb = rgb_of(emissive); + } + let glass = m + .transmission() + .map(|t| t.transmission_factor()) + .unwrap_or(0.0); + // OPAQUE ignores alpha, and MASK cuts rather than blends: + // only BLEND materials are see-through. + let alpha = match m.alpha_mode() { + gltf::material::AlphaMode::Blend => base[3], + gltf::material::AlphaMode::Opaque | gltf::material::AlphaMode::Mask => 1.0, + }; + ModelMaterial { + rgb, + alpha, + texture: pbr + .base_color_texture() + .map(|info| info.texture().source().index()), + material: Material { + roughness: pbr.roughness_factor().clamp(0.02, 1.0), + metalness: pbr.metallic_factor().clamp(0.0, 1.0), + glow: (glow * 2.0).min(8.0), + glass, + ior: m.ior().unwrap_or(1.5), + dispersion: 0.0, + }, + double_sided: m.double_sided(), + } + }) + .chain(std::iter::once(ModelMaterial { + // Primitives without a material. + rgb: 0xcccccc, + alpha: 1.0, + texture: None, + material: Material::default(), + double_sided: false, + })) + .collect(); + let default_material = materials.len() - 1; + + let mut model = Self { + triangles: Vec::new(), + materials, + images, + bounds: ([f32::INFINITY; 3], [f32::NEG_INFINITY; 3]), + }; + let scene = document + .default_scene() + .or_else(|| document.scenes().next()) + .ok_or("the model has no scene")?; + let identity: Mat4 = [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ]; + let mut stack: Vec<(gltf::Node, Mat4)> = + scene.nodes().map(|node| (node, identity)).collect(); + let mut visits = 0; + while let Some((node, parent)) = stack.pop() { + visits += 1; + if visits > MAX_NODE_VISITS { + return Err("the model's node tree is too large (or loops)".into()); + } + let world = multiply(&parent, &node.transform().matrix()); + if let Some(mesh) = node.mesh() { + for primitive in mesh.primitives() { + if primitive.mode() != gltf::mesh::Mode::Triangles { + continue; + } + // Only read attributes of the shape their meaning needs. + use gltf::accessor::{DataType, Dimensions}; + use gltf::mesh::Semantic; + let shaped = + |semantic: Semantic, dimensions: Dimensions, types: &[DataType]| { + primitive.get(&semantic).is_some_and(|a| { + a.dimensions() == dimensions && types.contains(&a.data_type()) + }) + }; + if !shaped(Semantic::Positions, Dimensions::Vec3, &[DataType::F32]) + || primitive.indices().is_some_and(|a| { + a.dimensions() != Dimensions::Scalar + || ![DataType::U8, DataType::U16, DataType::U32] + .contains(&a.data_type()) + }) + { + continue; + } + let normals_ok = shaped(Semantic::Normals, Dimensions::Vec3, &[DataType::F32]); + let uvs_ok = shaped( + Semantic::TexCoords(0), + Dimensions::Vec2, + &[DataType::F32, DataType::U8, DataType::U16], + ); + let reader = primitive.reader(|b| buffers.get(b.index()).map(|d| &d.0[..])); + let Some(positions) = reader.read_positions() else { + continue; + }; + let positions: Vec = + positions.map(|p| transform_point(&world, p)).collect(); + // Attributes must cover every vertex; a short one is dropped. + let normals: Option> = normals_ok + .then(|| reader.read_normals()) + .flatten() + .map(|n| n.map(|n| transform_normal(&world, n)).collect::>()) + .filter(|n| n.len() >= positions.len()); + let uvs: Option> = uvs_ok + .then(|| reader.read_tex_coords(0)) + .flatten() + .map(|t| t.into_f32().collect::>()) + .filter(|t| t.len() >= positions.len()); + let indices: Vec = match reader.read_indices() { + Some(indices) => indices.into_u32().map(|i| i as usize).collect(), + None => (0..positions.len()).collect(), + }; + let material = primitive.material().index().unwrap_or(default_material); + for tri in indices.chunks_exact(3) { + if tri.iter().any(|&i| i >= positions.len()) { + continue; + } + if model.triangles.len() >= MAX_TRIANGLES { + return Err(format!("the model has over {MAX_TRIANGLES} triangles")); + } + let pick = |list: &[V3]| [list[tri[0]], list[tri[1]], list[tri[2]]]; + let p = pick(&positions); + for v in &p { + for (axis, value) in v.iter().enumerate() { + model.bounds.0[axis] = model.bounds.0[axis].min(*value); + model.bounds.1[axis] = model.bounds.1[axis].max(*value); + } + } + model.triangles.push(Triangle { + positions: p, + normals: normals.as_deref().map(pick), + uvs: uvs.as_ref().map(|uv| [uv[tri[0]], uv[tri[1]], uv[tri[2]]]), + material, + }); + } + } + } + for child in node.children() { + if stack.len() >= MAX_NODE_VISITS { + return Err("the model's node tree is too large (or loops)".into()); + } + stack.push((child, world)); + } + } + if model.triangles.is_empty() { + return Err("the model has no triangle meshes".into()); + } + Ok(model) + } +} + +/// An `Arc`'d model for the scene's cache. +pub(crate) fn parse_shared(bytes: &[u8]) -> Result, String> { + Model::parse(bytes).map(Arc::new) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A binary glTF of one red, double-sided triangle under a node that + /// moves it up by 2 and doubles it. + pub(crate) fn triangle_glb() -> Vec { + let positions: [f32; 9] = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0]; + let bin: Vec = positions.iter().flat_map(|f| f.to_le_bytes()).collect(); + let json = format!( + r#"{{"asset":{{"version":"2.0"}},"scene":0,"scenes":[{{"nodes":[0]}}], + "nodes":[{{"mesh":0,"translation":[0,2,0],"scale":[2,2,2]}}], + "meshes":[{{"primitives":[{{"attributes":{{"POSITION":0}},"material":0}}]}}], + "materials":[{{"doubleSided":true,"pbrMetallicRoughness":{{"baseColorFactor":[1,0,0,1],"metallicFactor":0.5,"roughnessFactor":0.25}}}}], + "accessors":[{{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3","min":[0,0,0],"max":[1,1,0]}}], + "bufferViews":[{{"buffer":0,"byteLength":{len}}}], + "buffers":[{{"byteLength":{len}}}]}}"#, + len = bin.len() + ); + glb(&json, &bin) + } + + /// A binary glTF of `json` over `bin`. + fn glb(json: &str, bin: &[u8]) -> Vec { + let mut json = json.as_bytes().to_vec(); + while !json.len().is_multiple_of(4) { + json.push(b' '); + } + let total = 12 + 8 + json.len() + 8 + bin.len(); + let mut glb = Vec::with_capacity(total); + glb.extend_from_slice(b"glTF"); + glb.extend_from_slice(&2u32.to_le_bytes()); + glb.extend_from_slice(&(total as u32).to_le_bytes()); + glb.extend_from_slice(&(json.len() as u32).to_le_bytes()); + glb.extend_from_slice(b"JSON"); + glb.extend_from_slice(&json); + glb.extend_from_slice(&(bin.len() as u32).to_le_bytes()); + glb.extend_from_slice(b"BIN\0"); + glb.extend_from_slice(bin); + glb + } + + /// One triangle (3 positions, then `extra` bytes) and `rest` of the JSON. + fn triangle_with(extra: &[u8], rest: &str) -> Vec { + let positions: [f32; 9] = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0]; + let mut bin: Vec = positions.iter().flat_map(|f| f.to_le_bytes()).collect(); + bin.extend_from_slice(extra); + while !bin.len().is_multiple_of(4) { + bin.push(0); + } + let json = format!( + r#"{{"asset":{{"version":"2.0"}},"scene":0,"scenes":[{{"nodes":[0]}}], + "buffers":[{{"byteLength":{len}}}],{rest}}}"#, + len = bin.len() + ); + glb(&json, &bin) + } + + #[test] + fn looping_node_trees_are_refused_not_walked_forever() { + for children in ["[0]", "[0,0,0,0]"] { + let json = format!( + r#"{{"asset":{{"version":"2.0"}},"scene":0,"scenes":[{{"nodes":[0]}}],"nodes":[{{"children":{children}}}]}}"# + ); + let error = Model::parse(&glb(&json, &[])).unwrap_err(); + assert!(error.contains("loops"), "{error}"); + } + } + + #[test] + fn attributes_shorter_than_the_positions_are_dropped() { + // POSITION has 3 vertices, NORMAL only 1. + let normal: Vec = [0.0f32, 0.0, 1.0] + .iter() + .flat_map(|f| f.to_le_bytes()) + .collect(); + let model = Model::parse(&triangle_with( + &normal, + r#""nodes":[{"mesh":0}], + "meshes":[{"primitives":[{"attributes":{"POSITION":0,"NORMAL":1}}]}], + "accessors":[{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3","min":[0,0,0],"max":[1,1,0]}, + {"bufferView":1,"componentType":5126,"count":1,"type":"VEC3"}], + "bufferViews":[{"buffer":0,"byteLength":36},{"buffer":0,"byteOffset":36,"byteLength":12}]"#, + )) + .unwrap(); + assert!(model.triangles[0].normals.is_none()); + } + + #[test] + fn attributes_of_the_wrong_shape_are_skipped_not_read() { + // TEXCOORD_0 pointing at the VEC3 positions (the glTF reader asserts). + let model = Model::parse(&triangle_with( + &[], + r#""nodes":[{"mesh":0}], + "meshes":[{"primitives":[{"attributes":{"POSITION":0,"TEXCOORD_0":0}}]}], + "accessors":[{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3","min":[0,0,0],"max":[1,1,0]}], + "bufferViews":[{"buffer":0,"byteLength":36}]"#, + )) + .unwrap(); + assert!(model.triangles[0].uvs.is_none()); + } + + #[test] + fn embedded_images_stay_encoded_and_are_size_checked_when_drawn() { + // A tiny PNG declaring an image wider than any texture may be. + let mut png = Vec::new(); + ::image::RgbaImage::new(20_000, 1) + .write_to( + &mut std::io::Cursor::new(&mut png), + ::image::ImageFormat::Png, + ) + .unwrap(); + let rest = format!( + r#""nodes":[{{"mesh":0}}], + "meshes":[{{"primitives":[{{"attributes":{{"POSITION":0}},"material":0}}]}}], + "materials":[{{"pbrMetallicRoughness":{{"baseColorTexture":{{"index":0}}}}}}], + "textures":[{{"source":0}}], + "images":[{{"bufferView":1,"mimeType":"image/png"}}], + "accessors":[{{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3","min":[0,0,0],"max":[1,1,0]}}], + "bufferViews":[{{"buffer":0,"byteLength":36}},{{"buffer":0,"byteOffset":36,"byteLength":{len}}}]"#, + len = png.len() + ); + let model = Model::parse(&triangle_with(&png, &rest)).unwrap(); + let TextureData::Encoded(bytes) = &model.images[0] else { + panic!("decoded eagerly") + }; + assert_eq!(bytes.len(), png.len()); + let error = crate::image::decode_texture(&model.images[0]).unwrap_err(); + assert!(error.contains("20000x1"), "{error}"); + } + + #[test] + fn a_glb_parses_into_world_space_triangles_with_materials() { + let model = Model::parse(&triangle_glb()).unwrap(); + assert_eq!(model.triangles.len(), 1); + let tri = model.triangles[0]; + assert_eq!( + tri.positions, + [[0.0, 2.0, 0.0], [2.0, 2.0, 0.0], [0.0, 4.0, 0.0]] + ); + assert_eq!(model.bounds, ([0.0, 2.0, 0.0], [2.0, 4.0, 0.0])); + let material = &model.materials[tri.material]; + assert_eq!(material.rgb, 0xff0000); + assert!(material.double_sided); + assert_eq!(material.material.metalness, 0.5); + assert_eq!(material.material.roughness, 0.25); + } + + #[test] + fn garbage_is_an_error_not_a_panic() { + assert!(Model::parse(b"not a model").is_err()); + assert!(Model::parse(b"glTF\x02\0\0\0\x0c\0\0\0").is_err()); + } +} diff --git a/crates/visual/src/node.rs b/crates/visual/src/node.rs index d1be921fb5..87e8bff90d 100644 --- a/crates/visual/src/node.rs +++ b/crates/visual/src/node.rs @@ -262,6 +262,8 @@ pub struct ShaderNode { pub blend: crate::shader::Blend, /// Bloom strength on bright output (0 off). pub bloom: f32, + /// Image assets every pass samples as `iTexture0`-`3`. + pub textures: Vec, } pub const MIN_HEIGHT: f32 = 40.0; @@ -490,6 +492,7 @@ impl Parser { "blend", "bloom", "buffers", + "textures", // Before `blend`: true kept the alpha, false was opaque. "transparent", ])?; @@ -518,6 +521,21 @@ impl Parser { crate::shader::MAX_BUFFERS ))); } + let mut textures = Vec::new(); + if let Some(table) = props.get::>("textures")? { + for value in table.sequence_values::() { + match value? { + Value::String(name) => textures.push(name.to_string_lossy()), + _ => return Err(props.error("'textures' is a list of asset names")), + } + } + } + if textures.len() > crate::shader::MAX_TEXTURES { + return Err(props.error(&format!( + "takes at most {} textures", + crate::shader::MAX_TEXTURES + ))); + } let blend = match (props.string("blend")?, props.bool("transparent")?) { (Some(name), _) => crate::shader::Blend::parse(&name).ok_or_else(|| { props.error("'blend' is \"alpha\", \"light\", or \"opaque\"") @@ -532,9 +550,9 @@ impl Parser { let (name, value) = pair.map_err(|_| props.error("'inputs' maps names to numbers"))?; if !crate::shader::valid_input_name(&name) { - return Err(props.error(&format!( - "input '{name}' is not a usable GLSL name" - ))); + return Err( + props.error(&format!("input '{name}' is not a usable GLSL name")) + ); } inputs.push((name, value as f32)); } @@ -568,6 +586,7 @@ impl Parser { zoom: (zoom as f32).clamp(0.2, 5.0), blend, bloom: props.number("bloom")?.unwrap_or(0.0).clamp(0.0, 3.0) as f32, + textures, }) } "button" => { diff --git a/crates/visual/src/raster.rs b/crates/visual/src/raster.rs index 64e057fff7..2497349b3d 100644 --- a/crates/visual/src/raster.rs +++ b/crates/visual/src/raster.rs @@ -147,7 +147,15 @@ impl Scene { let bias = view.radius * 2e-3; let mut lines: Vec<[Sample; 3]> = Vec::new(); for line in &self.lines { - line_tris(&grid, &view, line.a, line.b, line.width, rgba(line.color), &mut lines); + line_tris( + &grid, + &view, + line.a, + line.b, + line.width, + rgba(line.color), + &mut lines, + ); } let (hints, labels) = self.annotations(camera, width, height); let mut dots = Vec::new(); @@ -607,8 +615,9 @@ fn resolve(grid: &Grid, color: &[[f32; 4]], width: usize, height: usize, ss: usi if alpha <= 0.0 { continue; } - let straight = - |ch: usize| ((sum[ch] / n / alpha).clamp(0.0, 1.0) * 255.0).round() as u8; + let straight = |ch: usize| { + ((sum[ch] / n / alpha).clamp(0.0, 1.0) * 255.0).round() as u8 + }; let o = x * 4; row[o] = straight(2); row[o + 1] = straight(1); @@ -655,8 +664,18 @@ mod tests { fn nearer_surfaces_hide_farther_ones_whatever_their_order() { for near_first in [true, false] { let mut s = Scene::default(); - let far = [[-2., -2., -1.], [2., -2., -1.], [2., 2., -1.], [-2., 2., -1.]]; - let near = [[-0.5, -0.5, 1.], [0.5, -0.5, 1.], [0.5, 0.5, 1.], [-0.5, 0.5, 1.]]; + let far = [ + [-2., -2., -1.], + [2., -2., -1.], + [2., 2., -1.], + [-2., 2., -1.], + ]; + let near = [ + [-0.5, -0.5, 1.], + [0.5, -0.5, 1.], + [0.5, 0.5, 1.], + [-0.5, 0.5, 1.], + ]; let faces = [(near, 0), (far, 1)]; let order: Vec<_> = if near_first { faces.to_vec() @@ -664,14 +683,25 @@ mod tests { faces.iter().rev().copied().collect() }; for (quad, chart) in order { - s.face(&quad, Color::token(Token::Chart(chart)), Some([0., 0., 1.]), true) - .unwrap(); + s.face( + &quad, + Color::token(Token::Chart(chart)), + Some([0., 0., 1.]), + true, + ) + .unwrap(); } let raster = s.render(camera(), 200.0, 200.0, 1.0, Quality::Draft, &colors); let center = pixel(&raster, 100, 100); - assert!(center[0] > 100 && center[2] < 60, "near red wins: {center:?}"); + assert!( + center[0] > 100 && center[2] < 60, + "near red wins: {center:?}" + ); let corner = pixel(&raster, 100, 30); - assert!(corner[2] > 100 && corner[0] < 60, "far blue shows around it: {corner:?}"); + assert!( + corner[2] > 100 && corner[0] < 60, + "far blue shows around it: {corner:?}" + ); // Nothing outside the geometry: transparent. assert_eq!(pixel(&raster, 1, 1)[3], 0); } @@ -681,9 +711,19 @@ mod tests { fn faces_crossing_the_camera_are_clipped_not_dropped() { let mut s = Scene::default(); // A long floor that runs from far away to behind the camera. - let floor = [[-1., -1., -40.], [1., -1., -40.], [1., -1., 40.], [-1., -1., 40.]]; - s.face(&floor, Color::token(Token::Chart(0)), Some([0., 1., 0.]), true) - .unwrap(); + let floor = [ + [-1., -1., -40.], + [1., -1., -40.], + [1., -1., 40.], + [-1., -1., 40.], + ]; + s.face( + &floor, + Color::token(Token::Chart(0)), + Some([0., 1., 0.]), + true, + ) + .unwrap(); // A marker the camera frames itself on, near the middle. s.face( &[[-0.2, 0., 0.], [0.2, 0., 0.], [0., 0.3, 0.]], @@ -733,11 +773,24 @@ mod timing { fn timing_a_furnished_room() { let mut s = Scene::default(); let quad = |s: &mut Scene, q: [V3; 4], c: u8| { - s.face(&q, Color::token(Token::Chart(c)), None, true).unwrap(); + s.face(&q, Color::token(Token::Chart(c)), None, true) + .unwrap(); }; - quad(&mut s, [[-4., 0., -3.], [4., 0., -3.], [4., 0., 3.], [-4., 0., 3.]], 0); - quad(&mut s, [[-4., 0., -3.], [4., 0., -3.], [4., 3., -3.], [-4., 3., -3.]], 1); - quad(&mut s, [[-4., 0., -3.], [-4., 0., 3.], [-4., 3., 3.], [-4., 3., -3.]], 2); + quad( + &mut s, + [[-4., 0., -3.], [4., 0., -3.], [4., 0., 3.], [-4., 0., 3.]], + 0, + ); + quad( + &mut s, + [[-4., 0., -3.], [4., 0., -3.], [4., 3., -3.], [-4., 3., -3.]], + 1, + ); + quad( + &mut s, + [[-4., 0., -3.], [-4., 0., 3.], [-4., 3., 3.], [-4., 3., -3.]], + 2, + ); // Spheres stand in for plants and lamps: smooth, many faces. for k in 0..8 { let at = [-3.0 + k as f32 * 0.8, 0.6, 1.0]; @@ -747,16 +800,25 @@ mod timing { let p = |i: u32, j: u32| { let t = std::f32::consts::TAU * i as f32 / lon as f32; let f = std::f32::consts::PI * j as f32 / lat as f32; - [at[0] + 0.4 * f.sin() * t.cos(), at[1] + 0.4 * f.cos(), at[2] + 0.4 * f.sin() * t.sin()] + [ + at[0] + 0.4 * f.sin() * t.cos(), + at[1] + 0.4 * f.cos(), + at[2] + 0.4 * f.sin() * t.sin(), + ] }; let q = [p(i, j), p(i + 1, j), p(i + 1, j + 1), p(i, j + 1)]; let n: Vec = q.iter().map(|c| sub(*c, at)).collect(); - s.smooth_face(&q, &n, Color::token(Token::Chart(3))).unwrap(); + s.smooth_face(&q, &n, Color::token(Token::Chart(3))) + .unwrap(); } } } let colors = |_: Color| [0.6, 0.6, 0.6, 1.0]; - let camera = Camera { yaw: 0.7, pitch: 0.4, zoom: 1.0 }; + let camera = Camera { + yaw: 0.7, + pitch: 0.4, + zoom: 1.0, + }; for quality in [Quality::Draft, Quality::Final] { let start = std::time::Instant::now(); let raster = s.render(camera, 736.0, 420.0, 2.0, quality, &colors); diff --git a/crates/visual/src/scene.rs b/crates/visual/src/scene.rs index 7b64f7eb63..77f101e080 100644 --- a/crates/visual/src/scene.rs +++ b/crates/visual/src/scene.rs @@ -5,13 +5,16 @@ //! other vector shape, so a 3D view stays transparent over glass and uses the //! theme's colors. +use std::sync::Arc; + use mlua::{Function, Table, UserData, UserDataMethods, Value}; use crate::draw::lua_error; use crate::node::{parse_color, value_text}; use crate::style::{Color, TextSize, Token}; -pub const MAX_FACES: usize = 30_000; +/// Faces a scene may draw (glTF models bring many). +pub const MAX_FACES: usize = 500_000; const MAX_ITEMS: usize = 20_000; pub type V3 = [f32; 3]; @@ -30,6 +33,76 @@ pub(crate) struct Face { /// and meshes light both sides. pub(crate) two_sided: bool, pub(crate) material: Material, + /// Texture coordinates per corner, for a textured face. + pub(crate) uvs: Option<[[f32; 2]; 4]>, + /// Index into the scene's textures. + pub(crate) texture: Option, +} + +/// An image a scene samples: an asset still encoded (PNG, JPEG, HDR), or +/// pixels a model brought already decoded. +#[derive(Debug, Clone, PartialEq)] +pub enum TextureData { + Encoded(Arc<[u8]>), + Rgba { + width: u32, + height: u32, + pixels: Arc<[u8]>, + }, +} + +/// One texture a scene uses; `key` names it stably across frames (so a +/// renderer decodes and uploads it once). +#[derive(Debug, Clone, PartialEq)] +pub struct TextureSource { + pub key: String, + pub data: TextureData, +} + +/// An environment map: an equirectangular image the scene reflects and +/// glass shows when nothing stands behind it. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Environment { + pub texture: u32, + pub intensity: f32, + /// Whether the scene stands in front of the map (else the chat shows + /// behind it, and the map only lights and reflects). + pub background: bool, +} + +/// A camera-facing image (a flare, a glint, a soft glow), drawn as added +/// light. +#[derive(Debug, Clone, PartialEq)] +pub struct Sprite { + pub at: V3, + /// Pixels across. + pub size: f32, + pub texture: Option, + pub color: Color, + pub glow: f32, +} + +/// The degrees `{x, y, z}` (or one number, about y) as a rotation, applied +/// about x, then y, then z. +fn rotation_of(value: Value) -> mlua::Result<[V3; 3]> { + let [x, y, z] = match value { + Value::Nil => return Ok([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]), + Value::Integer(n) => [0.0, n as f32, 0.0], + Value::Number(n) => [0.0, n as f32, 0.0], + other => v3(other, "rotate")?, + } + .map(f32::to_radians); + let (sx, cx, sy, cy, sz, cz) = (x.sin(), x.cos(), y.sin(), y.cos(), z.sin(), z.cos()); + // Rows of Rz * Ry * Rx. + Ok([ + [cz * cy, cz * sy * sx - sz * cx, cz * sy * cx + sz * sx], + [sz * cy, sz * sy * sx + cz * cx, sz * sy * cx - cz * sx], + [-sy, cy * sx, cy * cx], + ]) +} + +fn rotate(m: &[V3; 3], v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] } /// How a surface takes light. `glass` lets light through (refracting by @@ -70,6 +143,8 @@ pub struct FaceView<'a> { pub color: Color, pub two_sided: bool, pub material: Material, + pub uvs: Option<&'a [[f32; 2]]>, + pub texture: Option, } /// The framed orbit camera, for a renderer that does its own projection. @@ -122,6 +197,12 @@ pub struct Scene { pub time: f64, /// The material the primitive being added gives its faces. pub(crate) material: Material, + /// The texture the primitive being added gives its faces. + pub(crate) texture: Option, + pub(crate) textures: Vec, + pub(crate) environment: Option, + pub(crate) sprites: Vec, + pub(crate) assets: crate::assets::Assets, } /// The camera a scene is seen through: framed on its geometry and orbited. @@ -237,6 +318,68 @@ impl Scene { } } + /// A scene whose draw may use `assets` (textures, models). + pub fn with_assets(time: f64, assets: crate::assets::Assets) -> Self { + Self { + time, + assets, + ..Default::default() + } + } + + /// The texture keyed `key`, registered once; its index. + fn texture_index(&mut self, key: String, data: impl FnOnce() -> TextureData) -> u32 { + if let Some(i) = self.textures.iter().position(|t| t.key == key) { + return i as u32; + } + self.textures.push(TextureSource { key, data: data() }); + (self.textures.len() - 1) as u32 + } + + /// The asset `name` as a texture. + fn asset_texture(&mut self, name: &str) -> mlua::Result { + let bytes = self.assets.require(name).map_err(lua_error)?; + Ok(self.texture_index(format!("asset:{name}"), || TextureData::Encoded(bytes))) + } + + /// The asset `name` parsed as a glTF model, once per visual. + fn model(&self, name: &str) -> mlua::Result> { + let bytes = self.assets.require(name).map_err(lua_error)?; + let mut models = self + .assets + .models + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(model) = models.get(name) { + return Ok(model.clone()); + } + let model = crate::model::parse_shared(&bytes) + .map_err(|e| lua_error(format!("model '{name}': {e}")))?; + models.insert(name.to_owned(), model.clone()); + Ok(model) + } + + /// Give the face just added its texture coordinates. + fn uv_last(&mut self, uvs: &[[f32; 2]]) { + if let Some(face) = self.faces.last_mut() { + let mut out = [[0.0; 2]; 4]; + out[..uvs.len().min(4)].copy_from_slice(&uvs[..uvs.len().min(4)]); + face.uvs = Some(out); + } + } + + pub fn textures(&self) -> &[TextureSource] { + &self.textures + } + + pub fn environment(&self) -> Option { + self.environment + } + + pub fn sprites(&self) -> &[Sprite] { + &self.sprites + } + pub(crate) fn face( &mut self, corners: &[V3], @@ -278,12 +421,19 @@ impl Scene { color, two_sided, material: self.material, + uvs: None, + texture: self.texture, }); Ok(()) } /// A face of a curved solid, shaded from the surface's own normals. - pub(crate) fn smooth_face(&mut self, corners: &[V3], normals: &[V3], color: Color) -> mlua::Result<()> { + pub(crate) fn smooth_face( + &mut self, + corners: &[V3], + normals: &[V3], + color: Color, + ) -> mlua::Result<()> { let outward = normals.iter().fold([0.0; 3], |acc, n| add(acc, *n)); self.face(corners, color, Some(outward), false)?; let mut n = [[0.0; 3]; 4]; @@ -321,7 +471,7 @@ impl Scene { } fn items(&self) -> usize { - self.lines.len() + self.points.len() + self.labels.len() + self.lines.len() + self.points.len() + self.labels.len() + self.sprites.len() } fn check_items(&self) -> mlua::Result<()> { @@ -440,6 +590,8 @@ impl Scene { color: f.color, two_sided: f.two_sided, material: f.material, + uvs: f.uvs.as_ref().map(|uv| &uv[..f.n as usize]), + texture: f.texture, }) } @@ -563,7 +715,11 @@ fn v3(value: Value, what: &str) -> mlua::Result { /// A primitive's material options: roughness, metal, glow, glass (true or /// 0-1), ior, dispersion. fn material_of(options: &Table) -> mlua::Result { - let mut m = Material::default(); + material_over(Material::default(), options) +} + +/// `base` with whichever material options `options` sets. +fn material_over(mut m: Material, options: &Table) -> mlua::Result { let number = |key: &str| options.raw_get::>(key); if let Some(v) = number("roughness")? { m.roughness = v.clamp(0.02, 1.0); @@ -592,10 +748,37 @@ fn material_of(options: &Table) -> mlua::Result { Ok(m) } +/// A primitive's `texture = "asset name"`, registered with the scene. +fn texture_of(scene: &mut Scene, options: &Table) -> mlua::Result> { + match options.raw_get::>("texture")? { + Some(name) => scene.asset_texture(&name).map(Some), + None => Ok(None), + } +} + fn color_of(options: &Table, default: Token) -> mlua::Result { - Ok(parse_color(options.raw_get::("color")?) - .map_err(|e| lua_error(e.to_string()))? - .unwrap_or(Color::token(default))) + Ok(scene_color(options.raw_get::("color")?)?.unwrap_or(Color::token(default))) +} + +/// A scene color: a theme token like everywhere else, or, for physical +/// materials, `"#rrggbb"`. +pub(crate) fn scene_color(value: Value) -> mlua::Result> { + if let Value::String(s) = &value { + let text = s.to_string_lossy(); + if let Some(hex) = text.strip_prefix('#') { + let rgb = match hex.len() { + 6 => u32::from_str_radix(hex, 16).ok(), + 3 => u32::from_str_radix(hex, 16).ok().map(|v| { + let (r, g, b) = ((v >> 8) & 0xf, (v >> 4) & 0xf, v & 0xf); + ((r * 17) << 16) | ((g * 17) << 8) | (b * 17) + }), + _ => None, + } + .ok_or_else(|| lua_error(format!("'{text}' is not a #rrggbb color")))?; + return Ok(Some(Color::token(Token::Rgb(rgb)))); + } + } + parse_color(value).map_err(|e| lua_error(e.to_string())) } /// A ramp through `colors` (a list of tokens) at `t` in 0..1. @@ -636,6 +819,7 @@ impl UserData for Scene { fn add_methods>(methods: &mut M) { methods.add_method_mut("box", |_, s, o: Table| { s.material = material_of(&o)?; + s.texture = texture_of(s, &o)?; let at = v3(o.raw_get("at")?, "box at")?; let size = match o.raw_get::("size")? { Value::Nil => [1.0; 3], @@ -706,11 +890,13 @@ impl UserData for Scene { ]; for (quad, outward) in quads { s.face(&quad, color, Some(outward), false)?; + s.uv_last(&[[0., 1.], [1., 1.], [1., 0.], [0., 0.]]); } Ok(()) }); methods.add_method_mut("sphere", |_, s, o: Table| { s.material = material_of(&o)?; + s.texture = texture_of(s, &o)?; let at = v3(o.raw_get("at")?, "sphere at")?; let r = o.raw_get::>("radius")?.unwrap_or(0.5).abs(); let color = color_of(&o, Token::Accent)?; @@ -740,12 +926,22 @@ impl UserData for Scene { }; let normals: Vec = corners.iter().map(|c| sub(*c, at)).collect(); s.smooth_face(&corners, &normals, color)?; + let uv = |i: u32, j: u32| [i as f32 / lon as f32, j as f32 / lat as f32]; + let uvq = [uv(i, j), uv(i + 1, j), uv(i + 1, j + 1), uv(i, j + 1)]; + if j == 0 { + s.uv_last(&[uvq[0], uvq[2], uvq[3]]); + } else if j == lat - 1 { + s.uv_last(&[uvq[0], uvq[1], uvq[2]]); + } else { + s.uv_last(&uvq); + } } } Ok(()) }); methods.add_method_mut("cylinder", |_, s, o: Table| { s.material = material_of(&o)?; + s.texture = texture_of(s, &o)?; let at = v3(o.raw_get("at")?, "cylinder at")?; let r = o.raw_get::>("radius")?.unwrap_or(0.5).abs(); let h = o.raw_get::>("height")?.unwrap_or(1.0); @@ -765,7 +961,10 @@ impl UserData for Scene { .map(|c| [c[0] - at[0], 0.0, c[2] - at[2]]) .collect(); s.smooth_face(&quad, &normals, color)?; + let (u0, u1) = (i as f32 / n as f32, (i + 1) as f32 / n as f32); + s.uv_last(&[[u0, 1.0], [u1, 1.0], [u1, 0.0], [u0, 0.0]]); } + s.texture = None; let top: Vec = (0..n).map(|i| ring(i, h)).collect(); let bottom: Vec = (0..n).map(|i| ring(i, 0.0)).collect(); s.polygon(&top, color, Some([0.0, h.signum(), 0.0]), false)?; @@ -774,6 +973,7 @@ impl UserData for Scene { // A height field: y = f(x, z) over a grid, or `values` rows. methods.add_method_mut("surface", |_, s, o: Table| { s.material = material_of(&o)?; + s.texture = texture_of(s, &o)?; let range = |key: &str| -> mlua::Result<(f32, f32)> { match o.raw_get::>(key)? { Some(t) => Ok((t.raw_get::(1)?, t.raw_get::(2)?)), @@ -848,6 +1048,15 @@ impl UserData for Scene { } } else { s.face(&quad, color, Some([0.0, 1.0, 0.0]), true)?; + let (u0, u1) = ( + i as f32 / (cols - 1) as f32, + (i + 1) as f32 / (cols - 1) as f32, + ); + let (v0, v1) = ( + j as f32 / (rows - 1) as f32, + (j + 1) as f32 / (rows - 1) as f32, + ); + s.uv_last(&[[u0, v0], [u1, v0], [u1, v1], [u0, v1]]); } } } @@ -855,24 +1064,51 @@ impl UserData for Scene { }); methods.add_method_mut("mesh", |_, s, o: Table| { s.material = material_of(&o)?; + s.texture = texture_of(s, &o)?; let color = color_of(&o, Token::Accent)?; let vertices: Vec = o .raw_get::
("vertices")? .sequence_values::() .map(|v| v.and_then(|v| v3(v, "mesh vertex"))) .collect::>()?; + let uvs: Option> = match o.raw_get::>("uvs")? { + Some(list) => Some( + list.sequence_values::
() + .map(|uv| { + let uv = uv?; + Ok([uv.raw_get::(1)?, uv.raw_get::(2)?]) + }) + .collect::>()?, + ), + None => None, + }; for face in o.raw_get::
("faces")?.sequence_values::
() { - let corners: Vec = face? + let indices: Vec = face? .sequence_values::() - .map(|i| { - i.and_then(|i| { - vertices.get(i.wrapping_sub(1)).copied().ok_or_else(|| { - lua_error(format!("mesh face index {i} is out of range")) - }) + .collect::>()?; + let corners: Vec = indices + .iter() + .map(|&i| { + vertices.get(i.wrapping_sub(1)).copied().ok_or_else(|| { + lua_error(format!("mesh face index {i} is out of range")) }) }) .collect::>()?; - s.polygon(&corners, color, None, true)?; + match &uvs { + Some(uvs) if corners.len() >= 3 => { + let uv = |i: usize| { + uvs.get(indices[i].wrapping_sub(1)) + .copied() + .unwrap_or([0.0; 2]) + }; + // Fan it, so every triangle keeps its own coordinates. + for k in 1..corners.len() - 1 { + s.face(&[corners[0], corners[k], corners[k + 1]], color, None, true)?; + s.uv_last(&[uv(0), uv(k), uv(k + 1)]); + } + } + _ => s.polygon(&corners, color, None, true)?, + } } Ok(()) }); @@ -967,6 +1203,134 @@ impl UserData for Scene { Ok(()) }, ); + // A glTF model from the visual's assets: placed at `at`, turned by + // `rotate` (degrees), scaled by `scale`, or centered and sized so its + // longest side is `fit`. Its own materials and textures apply; any + // material option or `color` given here overrides them. + methods.add_method_mut("model", |_, s, o: Table| { + let name = o + .raw_get::>("asset")? + .ok_or_else(|| lua_error("s:model needs asset = \"name.glb\""))?; + let model = s.model(&name)?; + let at = match o.raw_get::("at")? { + Value::Nil => [0.0; 3], + other => v3(other, "model at")?, + }; + let mut size = match o.raw_get::("scale")? { + Value::Nil => [1.0; 3], + Value::Integer(n) => [n as f32; 3], + Value::Number(n) => [n as f32; 3], + other => v3(other, "model scale")?, + }; + let (lo, hi) = model.bounds; + let mut origin = [0.0; 3]; + if let Some(fit) = o.raw_get::>("fit")? { + let extent = (0..3).map(|i| hi[i] - lo[i]).fold(1e-6f32, f32::max); + size = scale(size, fit.abs() / extent); + origin = scale(add(lo, hi), 0.5); + } + let turn = rotation_of(o.raw_get("rotate")?)?; + let color = scene_color(o.raw_get::("color")?)?; + let texture = texture_of(s, &o)?; + let place = |p: V3| { + let p = sub(p, origin); + add( + at, + rotate(&turn, [p[0] * size[0], p[1] * size[1], p[2] * size[2]]), + ) + }; + // Normals scale inversely, so a squashed model still shades right. + let orient = |n: V3| { + rotate( + &turn, + normalize([n[0] / size[0], n[1] / size[1], n[2] / size[2]]), + ) + }; + let mut images: Vec> = vec![None; model.images.len()]; + for tri in &model.triangles { + let source = &model.materials[tri.material]; + s.material = material_over(source.material, &o)?; + s.texture = texture.or_else(|| { + let image = source.texture.filter(|i| *i < images.len())?; + Some(*images[image].get_or_insert_with(|| { + let data = model.images[image].clone(); + s.texture_index(format!("model:{name}:{image}"), || data) + })) + }); + if tri.uvs.is_none() { + s.texture = None; + } + let color = color.unwrap_or(Color { + token: Token::Rgb(source.rgb), + mix: None, + alpha: source.alpha, + }); + let corners = tri.positions.map(place); + match tri.normals { + Some(normals) => s.smooth_face(&corners, &normals.map(orient), color)?, + None => s.face(&corners, color, None, source.double_sided)?, + } + if let Some(face) = s.faces.last_mut() { + face.two_sided = source.double_sided; + } + if let Some(uvs) = tri.uvs { + s.uv_last(&uvs); + } + } + s.texture = None; + Ok(()) + }); + // An environment map (an equirectangular image asset, ideally HDR): + // what metal and glass reflect, and the light around the scene. + methods.add_method_mut("environment", |_, s, o: Table| { + let name = o + .raw_get::>("asset")? + .ok_or_else(|| lua_error("s:environment needs asset = \"name.hdr\""))?; + let texture = s.asset_texture(&name)?; + s.environment = Some(Environment { + texture, + intensity: o + .raw_get::>("intensity")? + .unwrap_or(1.0) + .clamp(0.0, 16.0), + background: o.raw_get::>("background")?.unwrap_or(false), + }); + Ok(()) + }); + // A camera-facing glow (a flare, a glint), `size` pixels across: an + // image asset, or a soft round light without one. + methods.add_method_mut("sprite", |_, s, (at, o): (Value, Option
)| { + s.check_items()?; + let (at, o) = options_form(at, o)?; + let at = v3(at, "sprite")?; + let sprite = match o { + Some(o) => Sprite { + at, + size: o + .raw_get::>("size")? + .unwrap_or(48.0) + .clamp(1.0, 2048.0), + texture: texture_of(s, &o)?.or(match o.raw_get::>("asset")? { + Some(name) => Some(s.asset_texture(&name)?), + None => None, + }), + color: color_of(&o, Token::Accent)?, + glow: o + .raw_get::>("glow")? + .unwrap_or(1.0) + .clamp(0.0, 16.0), + }, + None => Sprite { + at, + size: 48.0, + texture: None, + color: Color::token(Token::Accent), + glow: 1.0, + }, + }; + s.sprites.push(sprite); + Ok(()) + }); // A floor grid on y = 0 in hairlines. methods.add_method_mut("grid", |_, s, o: Option
| { let (size, step, y) = match &o { diff --git a/crates/visual/src/shader.rs b/crates/visual/src/shader.rs index 84e8ca88b9..dd0359521c 100644 --- a/crates/visual/src/shader.rs +++ b/crates/visual/src/shader.rs @@ -18,6 +18,9 @@ //! current frame, and a pass sees its own previous frame (feedback: trails, //! simulations, accumulation). `iChannelResolution[i]` is each channel's //! size, and `iNoise` is a tiling 256x256 texture of independent random RGBA. +//! +//! Image assets shipped with the visual (`textures = { "rock.jpg" }`) are +//! `iTexture0`-`iTexture3`, repeating, with mipmaps. /// At most this many named inputs (four vec4 slots). pub const MAX_INPUTS: usize = 16; @@ -25,6 +28,8 @@ pub const MAX_INPUTS: usize = 16; pub const MAX_SHADER_BYTES: usize = 64 * 1024; /// Buffer passes before the image pass. pub const MAX_BUFFERS: usize = 4; +/// Image assets a shader may sample. +pub const MAX_TEXTURES: usize = 4; /// How a shader's output sits on the chat. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] @@ -104,12 +109,20 @@ pub fn program(code: &str, inputs: &[String]) -> (String, usize) { layout(set = 0, binding = 5) uniform texture2D zeron_noise;\n\ layout(set = 0, binding = 6) uniform sampler zeron_clamp;\n\ layout(set = 0, binding = 7) uniform sampler zeron_repeat;\n\ + layout(set = 0, binding = 8) uniform texture2D zeron_texture0;\n\ + layout(set = 0, binding = 9) uniform texture2D zeron_texture1;\n\ + layout(set = 0, binding = 10) uniform texture2D zeron_texture2;\n\ + layout(set = 0, binding = 11) uniform texture2D zeron_texture3;\n\ layout(location = 0) out vec4 zeron_color;\n\ #define iChannel0 sampler2D(zeron_channel0, zeron_clamp)\n\ #define iChannel1 sampler2D(zeron_channel1, zeron_clamp)\n\ #define iChannel2 sampler2D(zeron_channel2, zeron_clamp)\n\ #define iChannel3 sampler2D(zeron_channel3, zeron_clamp)\n\ #define iNoise sampler2D(zeron_noise, zeron_repeat)\n\ + #define iTexture0 sampler2D(zeron_texture0, zeron_repeat)\n\ + #define iTexture1 sampler2D(zeron_texture1, zeron_repeat)\n\ + #define iTexture2 sampler2D(zeron_texture2, zeron_repeat)\n\ + #define iTexture3 sampler2D(zeron_texture3, zeron_repeat)\n\ #define iResolution (zeron_resolution.xyz)\n\ #define iTime (zeron_time.x)\n\ #define iTimeDelta (zeron_time.y)\n\ @@ -117,7 +130,10 @@ pub fn program(code: &str, inputs: &[String]) -> (String, usize) { ); for (i, name) in inputs.iter().enumerate().take(MAX_INPUTS) { let lane = ["x", "y", "z", "w"][i % 4]; - header.push_str(&format!("#define {name} (zeron_inputs[{}].{lane})\n", i / 4)); + header.push_str(&format!( + "#define {name} (zeron_inputs[{}].{lane})\n", + i / 4 + )); } let lines = header.lines().count(); // Shadertoy sources often carry an ES precision statement; GLSL 4.50 @@ -169,6 +185,10 @@ pub fn valid_input_name(name: &str) -> bool { | "iChannel3" | "iChannelResolution" | "iNoise" + | "iTexture0" + | "iTexture1" + | "iTexture2" + | "iTexture3" | "main" ) } diff --git a/crates/visual/src/style.rs b/crates/visual/src/style.rs index c0cc82f02a..8cd13c6ee6 100644 --- a/crates/visual/src/style.rs +++ b/crates/visual/src/style.rs @@ -26,6 +26,9 @@ pub enum Token { Border, /// Categorical series colors, `chart1` … `chart6` (0-based here). Chart(u8), + /// A physical color (`0xRRGGBB`): only 3D scenes take these, for the + /// materials of real objects (wood, foliage, a model's own paint). + Rgb(u32), } pub const CHART_SERIES: u8 = 6; diff --git a/crates/visual/src/tests.rs b/crates/visual/src/tests.rs index f9982156f4..6a02893ec9 100644 --- a/crates/visual/src/tests.rs +++ b/crates/visual/src/tests.rs @@ -337,3 +337,105 @@ fn shaders_default_to_light_and_keep_the_old_transparent_flag() { assert_eq!(blend_of("transparent = false,"), Blend::Opaque); assert_eq!(blend_of("blend = \"opaque\","), Blend::Opaque); } + +fn png(width: u32, height: u32) -> Vec { + let mut out = Vec::new(); + ::image::RgbaImage::from_fn(width, height, |x, y| { + ::image::Rgba([(x * 30) as u8, (y * 30) as u8, 200, 255]) + }) + .write_to( + &mut std::io::Cursor::new(&mut out), + ::image::ImageFormat::Png, + ) + .unwrap(); + out +} + +#[test] +fn scenes_place_models_textures_environments_and_sprites_from_assets() { + let assets = Assets::new( + [ + ( + "tri.glb".to_owned(), + crate::model::tests::triangle_glb().into(), + ), + ("wood.png".to_owned(), png(4, 4).into()), + ("sky.png".to_owned(), png(8, 4).into()), + ] + .into_iter() + .collect(), + ); + let source = r##" + return ui.scene { height = 300, draw = function(s) + s:environment { asset = "sky.png", intensity = 1.5 } + s:model { asset = "tri.glb", fit = 1, rotate = { 0, 90, 0 } } + s:model { asset = "tri.glb", at = { 3, 0, 0 }, color = "#00ff00", metal = 1 } + s:box { at = { 0, -1, 0 }, texture = "wood.png" } + s:sprite { at = { 0, 2, 0 }, size = 64, color = "#ffeedd", glow = 2 } + end } + "##; + let mut visual = Visual::load(source, 1).unwrap().with_assets(assets.clone()); + let tree = visual.render().unwrap(); + let NodeKind::Scene(node) = &tree.kind else { + panic!() + }; + let scene = visual.draw_scene(node.draw, 0.0).unwrap(); + assert_eq!(scene.face_count(), 2 + 6); + assert_eq!(scene.textures().len(), 2); + assert_eq!(scene.environment().map(|e| e.intensity), Some(1.5)); + assert_eq!(scene.sprites().len(), 1); + let faces: Vec = scene.face_views().collect(); + // `fit` centers the model and sizes its longest side; the rotation turns + // the triangle's plane from z = 0 onto x = 0. + let fitted = &faces[0]; + for corner in fitted.corners { + assert!(corner[0].abs() < 1e-5, "{corner:?}"); + assert!(corner[1].abs() <= 0.5 + 1e-5 && corner[2].abs() <= 0.5 + 1e-5); + } + assert_eq!(fitted.color.token, Token::Rgb(0xff0000)); + assert!(fitted.two_sided); + assert_eq!(faces[1].color.token, Token::Rgb(0x00ff00)); + assert_eq!(faces[1].material.metalness, 1.0); + assert_eq!(faces[1].material.roughness, 0.25); + let boxed = &faces[2]; + assert_eq!(boxed.texture, Some(1)); + assert_eq!(boxed.uvs.map(|uv| uv.len()), Some(4)); + // The model is parsed once per visual, however often the scene draws. + visual.draw_scene(node.draw, 1.0).unwrap(); + assert_eq!(assets.models.lock().unwrap().len(), 1); + check_with_assets(source, 1, assets).unwrap(); +} + +#[test] +fn missing_and_broken_assets_fail_the_check_with_their_names() { + let script = + |body: &str| format!("return ui.scene {{ height = 200, draw = function(s) {body} end }}"); + let error = check(&script(r#"s:model { asset = "prism.glb" }"#), 1).unwrap_err(); + assert!( + error.message.contains("no asset named 'prism.glb'"), + "{}", + error.message + ); + let assets = Assets::new( + [ + ("bad.png".to_owned(), b"nope".to_vec().into()), + ("bad.glb".to_owned(), b"nope".to_vec().into()), + ] + .into_iter() + .collect(), + ); + let error = check_with_assets( + &script(r#"s:box { at = {0,0,0}, texture = "bad.png" }"#), + 1, + assets.clone(), + ) + .unwrap_err(); + assert!(error.message.contains("bad.png"), "{}", error.message); + let error = + check_with_assets(&script(r#"s:model { asset = "bad.glb" }"#), 1, assets).unwrap_err(); + assert!( + error.message.contains("model 'bad.glb'"), + "{}", + error.message + ); +} diff --git a/docs/mcp.md b/docs/mcp.md index 99dd6f9ad9..9b3021ba6c 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -125,7 +125,7 @@ name (default: the local engine's device). | `interrupt_chat` | `QueueCommand` Interrupt | | `respond_to_input` | `QueueCommand` RespondInput | | `archive_chat` | `Mutate setChatArchived` | -| `show_visual` | `PublishVisual {chatId, title, script}` (origin chat only) | +| `show_visual` | `PublishVisual {chatId, title, script, assets}` (origin chat only; assets base64 by name, read by the MCP server from the agent's cwd) | Watch streams are the engine's only read surface (there is no one-shot "get transcript" RPC); a snapshot is "subscribe, take the first item, drop" — drop @@ -189,11 +189,14 @@ are theme tokens (`text`, `muted`, `faint`, `accent`, `on-accent`, `success`, `warning`, `danger`, `border`, `chart1`…`chart6`), text sizes are steps on the transcript's type scale, the font is always the interface font, and layout nodes have no fill, border, shadow, padding, or font props; using one is an error that -names the rule. 3D scenes are rendered in the runtime (`raster.rs`): Zeron -frames an orbit camera on the geometry, clips faces to the near plane, and -rasterizes them with a depth buffer (smooth-shaded curved solids, a sky/key/fill -light rig, depth-edge outlines, supersampled at rest), in parallel row bands, -into one transparent image. `ui.shader` takes Shadertoy-style GLSL +names the rule. 3D scenes render on the GPU (`zeron-visual-gpu`: physically +based materials, shadows, glass with dispersion, glTF models, textures, +environment maps, sprites and bloom), with the runtime's software rasterizer +(`raster.rs`) as the fallback where there's no GPU: Zeron frames an orbit +camera on the geometry and draws it into one transparent image. A visual's +files (`assets`: at most 16, 8 MB each, 10 MB together) are stored beside its +script as `{id}-asset-{name}` with an `{id}-assets.json` list, and travel to +other devices like attachments. `ui.shader` takes Shadertoy-style GLSL (`mainImage`, `iResolution`, `iTime`, `iMouse`, plus `iCamera` for an orbit camera, named float inputs bound to the script's state, `iNoise`, and up to four Shadertoy-style buffer passes read as `iChannel0`-`3`, with feedback from their From 5ae72231f28081768aa5b1589d5452eaa0fd6f47 Mon Sep 17 00:00:00 2001 From: Wing <44917547+wingleeio@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:52:00 -0700 Subject: [PATCH 4/4] Render desktop visual frames off the main thread and reuse GPU targets Desktop scenes and shaders rendered and read back inside GPUI's paint, blocking the main thread on the GPU every animated frame, so a heavy agent shader could stall the app. Paint now does only the cheap parts (input, uniforms, the scene's budgeted Lua draw) and hands the render to a background thread, one frame in flight per visual; the view shows the newest finished frame and repaints when the next lands. Scenes also allocated ~70 MB of render targets every frame. They now live in a SceneState kept per visual (desktop and iOS) and are remade only when the size changes; scenes and shaders keep their readback staging buffer too. The test room's steady frame went from ~8 ms to ~3 ms at 1440x720. Co-Authored-By: Claude Opus 5.5 --- crates/mobile/src/visual/mod.rs | 13 +- crates/ui/src/visuals.rs | 432 +++++++++++++++++++------------- crates/visual-gpu/src/lib.rs | 27 +- crates/visual-gpu/src/scene.rs | 172 ++++++++----- crates/visual-gpu/src/shader.rs | 4 +- 5 files changed, 410 insertions(+), 238 deletions(-) diff --git a/crates/mobile/src/visual/mod.rs b/crates/mobile/src/visual/mod.rs index 50e090c839..199a492d51 100644 --- a/crates/mobile/src/visual/mod.rs +++ b/crates/mobile/src/visual/mod.rs @@ -498,6 +498,8 @@ struct SurfaceState { reads_pointer: bool, orbit: Option, shader: Option, + /// A scene's GPU targets, kept while its size holds. + scene_gpu: Option, shader_frame: u32, shader_time: f64, scene: Option<(u64, zeron_visual::Scene)>, @@ -529,6 +531,7 @@ impl SurfaceState { reads_pointer: false, orbit, shader: None, + scene_gpu: None, shader_frame: 0, shader_time: 0.0, scene: None, @@ -1136,7 +1139,15 @@ impl Worker { }; let pixel_scale = self.source.pixel_scale.min(MAX_GPU_SCALE); let (image, hints, labels) = - match zeron_visual_gpu::render_scene(scene, camera, w, h, pixel_scale, &rgba) { + match zeron_visual_gpu::render_scene( + scene, + camera, + w, + h, + pixel_scale, + &rgba, + &mut state.scene_gpu, + ) { Some(frame) => { let (hints, labels) = scene.annotations(camera, w, h); ( diff --git a/crates/ui/src/visuals.rs b/crates/ui/src/visuals.rs index a07ce718ba..56e5446d3a 100644 --- a/crates/ui/src/visuals.rs +++ b/crates/ui/src/visuals.rs @@ -78,15 +78,19 @@ struct Local { draw_error: Option, } -/// A shader's last frame and what it was drawn with. +/// A shader's newest finished frame, and the next one in the works. Frames +/// render on a background thread; the view shows the newest one meanwhile. +#[derive(Default)] struct ShaderCache { - uniforms: zeron_visual::shader::ShaderUniforms, - size: (u32, u32), - image: Arc, + /// The newest frame and the uniforms it was drawn with. + shown: Option<(zeron_visual::shader::ShaderUniforms, Arc)>, + /// Frames started, and the wall time the last one was started at. frame: u32, last_time: f64, - /// Buffers carried between frames (feedback, simulations). + /// Buffers carried between frames (feedback, simulations); away on the + /// render thread while a frame renders. gpu: Option, + rendering: bool, } struct CanvasCache { @@ -96,14 +100,21 @@ struct CanvasCache { commands: Rc>, } +/// A scene's drawing, its newest finished image, and the next one in the +/// works (rendered on a background thread, like shaders). struct SceneCache { generation: u64, - scene: Rc, - /// The last image rendered from `scene`, and what it was rendered for. + /// Bumped with every new drawing, so an image of an older one is stale. + drawing: u64, + scene: Arc, + /// The newest image, and what it was rendered for. image: Option<(RasterKey, SceneImage)>, - /// How long the last supersampled render took: cheap scenes stay - /// supersampled while the camera moves. + /// How long the last supersampled software render took: cheap scenes + /// stay supersampled while the camera moves. final_cost: Option, + /// GPU targets kept between frames; away while a frame renders. + gpu: Option, + rendering: bool, } /// What a scene image depends on besides the geometry. @@ -116,13 +127,22 @@ struct RasterKey { pixel_scale: f32, quality: zeron_visual::Quality, palette: u64, + drawing: u64, } #[derive(Clone)] struct SceneImage { image: Arc, - hints: Rc>, - labels: Rc>, + hints: Arc>, + labels: Arc>, +} + +/// A rendered frame as an image GPUI can paint (off the main thread). +fn render_image(width: u32, height: u32, bgra: Vec) -> Option> { + let buffer = image::RgbaImage::from_raw(width, height, bgra)?; + Some(Arc::new(gpui::RenderImage::new([image::Frame::new( + buffer, + )]))) } #[derive(Clone, Copy)] @@ -368,6 +388,7 @@ fn chart_series(theme: &Theme) -> [Hsla; 6] { } /// Theme colors for one paint, resolved once. +#[derive(Clone)] struct Palette { text: Hsla, muted: Hsla, @@ -533,7 +554,7 @@ impl Local { let mut images = Vec::new(); self.shaders.retain(|key, cache| { kept(key) || { - images.push(cache.image.clone()); + images.extend(cache.shown.as_ref().map(|(_, image)| image.clone())); false } }); @@ -1626,7 +1647,11 @@ impl VisualFrame { .textures .iter() // A missing one keeps its slot (and samples as clear). - .map(|name| zeron_visual::TextureData::Encoded(assets.get(name).unwrap_or_else(|| Arc::from([])))) + .map(|name| { + zeron_visual::TextureData::Encoded( + assets.get(name).unwrap_or_else(|| Arc::from([])), + ) + }) .collect(); let (code, buffers, blend, bloom, zoom) = ( node.code.clone(), @@ -1637,10 +1662,10 @@ impl VisualFrame { ); // Buffers evolve every frame, so a shader with any keeps running. let moving = moving || !buffers.is_empty() && !cx.reduce_motion(); - let (local, time) = (self.local.clone(), self.time()); + let (local, time, entity) = (self.local.clone(), self.time(), cx.entity().downgrade()); el.child( canvas( - move |bounds, window, _| { + move |bounds, window, cx| { if moving { window.request_animation_frame(); } @@ -1655,7 +1680,10 @@ impl VisualFrame { // (negative otherwise). let device = |p: Point| { let at = p - bounds.origin; - [f32::from(at.x) * scale, size.1 as f32 - f32::from(at.y) * scale] + [ + f32::from(at.x) * scale, + size.1 as f32 - f32::from(at.y) * scale, + ] }; let mouse = match (state.presses.get(&key), state.pointers.get(&key)) { (Some(start), Some(now)) => { @@ -1668,88 +1696,112 @@ impl VisualFrame { } _ => [0.0, 0.0, -1.0, -1.0], }; - let buffers_run = !buffers.is_empty(); - let previous = state.shaders.get(&key); + let entry = state.shaders.entry(key.clone()).or_default(); // A still shader keeps the time it first showed. let shown_time = if animate { time } else { - previous.map_or(0.0, |p| f64::from(p.uniforms.time[0])) + entry + .shown + .as_ref() + .map_or(0.0, |(u, _)| f64::from(u.time[0])) + }; + let delta = if entry.frame == 0 { + 0.0 + } else { + (time - entry.last_time).max(0.0) }; - let frame = previous.map_or(0, |p| p.frame + 1); - let delta = previous.map_or(0.0, |p| (time - p.last_time).max(0.0)); let uniforms = zeron_visual::shader::ShaderUniforms { resolution: [size.0 as f32, size.1 as f32, scale, 0.0], mouse, camera: [yaw, orbit.pitch, zoom, 0.0], - time: [shown_time as f32, delta as f32, frame as f32, 0.0], + time: [shown_time as f32, delta as f32, entry.frame as f32, 0.0], inputs, channels: Default::default(), }; - let unchanged = !buffers_run && previous.is_some_and(|p| { - p.size == size - && p.uniforms.resolution == uniforms.resolution - && p.uniforms.mouse == uniforms.mouse - && p.uniforms.camera == uniforms.camera - && p.uniforms.inputs == uniforms.inputs - && (!animate || p.uniforms.time[0] == uniforms.time[0]) - }); - if unchanged { - return previous.map(|p| p.image.clone()); - } - let mut gpu_state = state.shaders.get_mut(&key).and_then(|p| p.gpu.take()); - let rendered = zeron_visual_gpu::render_shader( - &zeron_visual_gpu::ShaderSpec { - image: &code, - buffers: &buffers, - inputs: &names, - blend, - bloom, - textures: &textures, - }, - &uniforms, - size.0, - size.1, - &mut gpu_state, - ); - match rendered { - Ok(rendered) => { - let buffer = - image::RgbaImage::from_raw(rendered.width, rendered.height, rendered.bgra)?; - let image = - Arc::new(gpui::RenderImage::new([image::Frame::new(buffer)])); - let replaced = state.shaders.insert( - key.clone(), - ShaderCache { - uniforms, - size, - image: image.clone(), - frame, - last_time: time, - gpu: gpu_state, + let unchanged = buffers.is_empty() + && entry.shown.as_ref().is_some_and(|(p, _)| { + p.resolution == uniforms.resolution + && p.mouse == uniforms.mouse + && p.camera == uniforms.camera + && p.inputs == uniforms.inputs + && (!animate || p.time[0] == uniforms.time[0]) + }); + let shown = entry.shown.as_ref().map(|(_, image)| image.clone()); + // One frame in flight at a time; when it lands the view + // paints again and starts the next if anything moved. + if !unchanged && !entry.rendering { + entry.rendering = true; + entry.frame += 1; + entry.last_time = time; + let mut gpu_state = entry.gpu.take(); + let (code, buffers, names, textures) = ( + code.clone(), + buffers.clone(), + names.clone(), + textures.clone(), + ); + let render = cx.background_executor().spawn(async move { + let rendered = zeron_visual_gpu::render_shader( + &zeron_visual_gpu::ShaderSpec { + image: &code, + buffers: &buffers, + inputs: &names, + blend, + bloom, + textures: &textures, }, - ); - if let Some(old) = replaced { - let _ = window.drop_image(old.image); - } - state.draw_error = None; - Some(image) - } - Err(error) => { - state.draw_error = Some(error); - None - } + &uniforms, + size.0, + size.1, + &mut gpu_state, + ) + .and_then(|frame| { + render_image(frame.width, frame.height, frame.bgra) + .ok_or_else(|| "the shader's frame was malformed".to_owned()) + }); + (rendered, gpu_state) + }); + let (local, key, entity) = (local.clone(), key.clone(), entity.clone()); + cx.spawn(async move |cx| { + let (rendered, gpu_state) = render.await; + cx.update(|cx| { + let mut freed = Vec::new(); + let mut state = local.borrow_mut(); + match (state.shaders.get_mut(&key), rendered) { + (Some(entry), Ok(image)) => { + entry.rendering = false; + entry.gpu = gpu_state; + freed.extend( + entry + .shown + .replace((uniforms, image)) + .map(|(_, old)| old), + ); + state.draw_error = None; + } + (Some(entry), Err(error)) => { + entry.rendering = false; + entry.gpu = gpu_state; + state.draw_error = Some(error); + } + // The node left the tree meanwhile. + (None, Ok(image)) => freed.push(image), + (None, Err(_)) => {} + } + drop(state); + free_images(freed, cx); + let _ = entity.update(cx, |_, cx| cx.notify()); + }) + }) + .detach(); } + shown }, move |bounds, image, window, _| { if let Some(image) = image { - let _ = window.paint_image( - bounds, - gpui::Corners::default(), - image, - 0, - false, - ); + let _ = + window.paint_image(bounds, gpui::Corners::default(), image, 0, false); } }, ) @@ -1791,6 +1843,7 @@ impl VisualFrame { let moving = (spin != 0.0 && orbit.drag.is_none()) || node.animate; let dragging = orbit.drag.is_some(); let (local, generation, time) = (self.local.clone(), self.generation, self.time()); + let entity = cx.entity().downgrade(); let (draw, animate) = (node.draw, node.animate); let paint_palette = Rc::new(Palette::new(Theme::of(cx))); let mut el = div() @@ -1805,56 +1858,54 @@ impl VisualFrame { let render_palette = paint_palette.clone(); el.child( canvas( - move |bounds, window, _| { + move |bounds, window, cx| { let paint_palette = &render_palette; if moving { window.request_animation_frame(); } let mut state = local.borrow_mut(); - let cached = state + let fresh = state .scenes .get(&key) - .filter(|c| !animate && c.generation == generation) - .map(|c| c.scene.clone()); - let scene = match cached { - Some(scene) => scene, - None => { - drop(state); - let drawn = runtime.borrow_mut().draw_scene(draw, time); - state = local.borrow_mut(); - match drawn { - Ok(scene) => { - let scene = Rc::new(scene); - // A new drawing replaces the old one's image. - let previous = state.scenes.insert( - key.clone(), - SceneCache { - generation, - scene: scene.clone(), - image: None, - final_cost: None, - }, - ); - if let Some(previous) = previous { - // An animated scene redraws every - // frame; its render cost carries over. - if let Some(entry) = state.scenes.get_mut(&key) { - entry.final_cost = previous.final_cost; - } - if let Some((_, old)) = previous.image { - let _ = window.drop_image(old.image); - } + .is_some_and(|c| !animate && c.generation == generation); + if !fresh { + drop(state); + let drawn = runtime.borrow_mut().draw_scene(draw, time); + state = local.borrow_mut(); + match drawn { + Ok(scene) => { + let scene = Arc::new(scene); + // A new drawing keeps the old image up (and the + // GPU targets) until its own frame lands. + match state.scenes.get_mut(&key) { + Some(entry) => { + entry.generation = generation; + entry.drawing += 1; + entry.scene = scene; + } + None => { + state.scenes.insert( + key.clone(), + SceneCache { + generation, + drawing: 0, + scene, + image: None, + final_cost: None, + gpu: None, + rendering: false, + }, + ); } - state.draw_error = None; - scene - } - Err(error) => { - state.draw_error = Some(error.message); - return None; } + state.draw_error = None; + } + Err(error) => { + state.draw_error = Some(error.message); + return None; } } - }; + } // Always at full density. While the camera moves, a scene // too heavy to supersample within a frame drops to one // sample per pixel; at rest it is always supersampled. @@ -1866,12 +1917,14 @@ impl VisualFrame { .is_some_and(|cost| cost < std::time::Duration::from_millis(8)); // Quality only steers the software renderer; on the GPU // a finished drag mustn't render the same frame again. - let quality = if (moving || dragging) && !cheap && !zeron_visual_gpu::available() { - zeron_visual::Quality::Draft - } else { - zeron_visual::Quality::Final - }; + let quality = + if (moving || dragging) && !cheap && !zeron_visual_gpu::available() { + zeron_visual::Quality::Draft + } else { + zeron_visual::Quality::Final + }; let size = (f32::from(bounds.size.width), f32::from(bounds.size.height)); + let entry = state.scenes.get_mut(&key)?; let raster_key = RasterKey { yaw: camera.yaw, pitch: camera.pitch, @@ -1880,58 +1933,101 @@ impl VisualFrame { pixel_scale, quality, palette: palette_key(paint_palette), + drawing: entry.drawing, }; - let entry = state.scenes.get_mut(&key)?; - let image = match &entry.image { - Some((cached_key, image)) if *cached_key == raster_key => image.clone(), - _ => { - let palette = paint_palette.clone(); + let shown = entry.image.as_ref().map(|(_, image)| image.clone()); + // One frame in flight at a time, on a background thread; + // when it lands the view paints again. + if entry.image.as_ref().map(|(k, _)| *k) != Some(raster_key) && !entry.rendering + { + entry.rendering = true; + let (scene, mut gpu_state) = (entry.scene.clone(), entry.gpu.take()); + let palette = (**paint_palette).clone(); + let render = cx.background_executor().spawn(async move { let rgba = move |color| { let rgba = palette.color(color).to_rgb(); [rgba.r, rgba.g, rgba.b, rgba.a] }; // On the GPU when there is one (materials, shadows, // glass, bloom); the software renderer otherwise. - let (width, height, bgra, hints, labels) = match zeron_visual_gpu::render_scene( - &scene, - camera, - size.0, - size.1, - pixel_scale, - &rgba, - ) { - Some(frame) => { - let (hints, labels) = scene.annotations(camera, size.0, size.1); - (frame.width, frame.height, frame.bgra, hints, labels) - } - None => { - let started = Instant::now(); - let raster = scene.render( - camera, - size.0, - size.1, - pixel_scale, - quality, - &rgba, - ); - if quality == zeron_visual::Quality::Final { - entry.final_cost = Some(started.elapsed()); + let started = Instant::now(); + let mut cost = None; + let (width, height, bgra, hints, labels) = + match zeron_visual_gpu::render_scene( + &scene, + camera, + size.0, + size.1, + pixel_scale, + &rgba, + &mut gpu_state, + ) { + Some(frame) => { + let (hints, labels) = + scene.annotations(camera, size.0, size.1); + (frame.width, frame.height, frame.bgra, hints, labels) + } + None => { + let raster = scene.render( + camera, + size.0, + size.1, + pixel_scale, + quality, + &rgba, + ); + if quality == zeron_visual::Quality::Final { + cost = Some(started.elapsed()); + } + ( + raster.width, + raster.height, + raster.bgra, + raster.hints, + raster.labels, + ) + } + }; + let image = render_image(width, height, bgra).map(|image| SceneImage { + image, + hints: Arc::new(hints), + labels: Arc::new(labels), + }); + (image, gpu_state, cost) + }); + let (local, key, entity) = (local.clone(), key.clone(), entity.clone()); + cx.spawn(async move |cx| { + let (image, gpu_state, cost) = render.await; + cx.update(|cx| { + let mut freed = Vec::new(); + let mut state = local.borrow_mut(); + match state.scenes.get_mut(&key) { + Some(entry) => { + entry.rendering = false; + entry.gpu = gpu_state; + if cost.is_some() { + entry.final_cost = cost; + } + if let Some(image) = image { + freed.extend( + entry + .image + .replace((raster_key, image)) + .map(|(_, old)| old.image), + ); + } } - (raster.width, raster.height, raster.bgra, raster.hints, raster.labels) + // The node left the tree meanwhile. + None => freed.extend(image.map(|image| image.image)), } - }; - let buffer = image::RgbaImage::from_raw(width, height, bgra)?; - let image = SceneImage { - image: Arc::new(gpui::RenderImage::new([image::Frame::new(buffer)])), - hints: Rc::new(hints), - labels: Rc::new(labels), - }; - if let Some((_, old)) = entry.image.replace((raster_key, image.clone())) { - let _ = window.drop_image(old.image); - } - image - } - }; + drop(state); + free_images(freed, cx); + let _ = entity.update(cx, |_, cx| cx.notify()); + }) + }) + .detach(); + } + let image = shown?; let hovered = state.pointers.get(&pointer_key).and_then(|p| { let at = *p - bounds.origin; let (x, y) = (f32::from(at.x), f32::from(at.y)); diff --git a/crates/visual-gpu/src/lib.rs b/crates/visual-gpu/src/lib.rs index 5a31e29b66..96348cc1e5 100644 --- a/crates/visual-gpu/src/lib.rs +++ b/crates/visual-gpu/src/lib.rs @@ -13,7 +13,7 @@ mod scene; mod shader; mod textures; -pub use scene::render_scene; +pub use scene::{SceneState, render_scene}; pub use shader::{ShaderSpec, ShaderState, render_shader}; /// One rendered frame: straight-alpha BGRA, top row first. @@ -139,26 +139,31 @@ impl Gpu { }) } - /// Read a rendered texture back as straight-alpha BGRA. + /// Read a rendered texture back as straight-alpha BGRA, through a + /// staging buffer kept in `staging` between frames. pub(crate) fn read_back( &self, mut encoder: wgpu::CommandEncoder, texture: &wgpu::Texture, - opaque: bool, + staging: &mut Option, ) -> Frame { let (width, height) = (texture.width(), texture.height()); let row = (width * 4).div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; - let buffer = self.device.create_buffer(&wgpu::BufferDescriptor { - label: Some("visual readback"), - size: u64::from(row * height), - usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, - mapped_at_creation: false, - }); + let size = u64::from(row * height); + if staging.as_ref().is_none_or(|b| b.size() != size) { + *staging = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("visual readback"), + size, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + } + let buffer = staging.as_ref().expect("made above"); encoder.copy_texture_to_buffer( texture.as_image_copy(), wgpu::TexelCopyBufferInfo { - buffer: &buffer, + buffer, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), @@ -185,7 +190,7 @@ impl Gpu { d[0] = s[2]; d[1] = s[1]; d[2] = s[0]; - d[3] = if opaque { 255 } else { s[3] }; + d[3] = s[3]; } } drop(mapped); diff --git a/crates/visual-gpu/src/scene.rs b/crates/visual-gpu/src/scene.rs index 493292d07d..1a70c25fc2 100644 --- a/crates/visual-gpu/src/scene.rs +++ b/crates/visual-gpu/src/scene.rs @@ -1057,9 +1057,70 @@ fn geometry(scene: &Scene, frame: &CameraFrame, rgba: &dyn Fn(Color) -> [f32; 4] out } +/// One scene's render targets between frames (about 70 MB at a Retina +/// size): made once, then reused while the size holds. +pub struct SceneState { + size: (u32, u32), + shadow_map: wgpu::Texture, + msaa_color: wgpu::Texture, + msaa_depth: wgpu::Texture, + opaque_resolved: wgpu::Texture, + lit: wgpu::Texture, + bloom_a: wgpu::Texture, + bloom_b: wgpu::Texture, + output: wgpu::Texture, + dummy_depth: wgpu::Texture, + staging: Option, +} + +impl SceneState { + fn new(gpu: &crate::Gpu, (w, h): (u32, u32)) -> Self { + use wgpu::TextureUsages as U; + let device = &gpu.device; + let texture = |label: &str, size: (u32, u32), format, samples, usage| { + device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: size.0, + height: size.1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: samples, + dimension: wgpu::TextureDimension::D2, + format, + usage, + view_formats: &[], + }) + }; + let sampled = U::RENDER_ATTACHMENT | U::TEXTURE_BINDING; + let half = ((w / 2).max(1), (h / 2).max(1)); + Self { + size: (w, h), + shadow_map: texture( + "scene shadow map", + (SHADOW_SIZE, SHADOW_SIZE), + DEPTH, + 1, + sampled, + ), + msaa_color: texture("scene color", (w, h), HDR, SAMPLES, U::RENDER_ATTACHMENT), + msaa_depth: texture("scene depth", (w, h), DEPTH, SAMPLES, sampled), + opaque_resolved: texture("scene behind glass", (w, h), HDR, 1, sampled), + lit: texture("scene lit", (w, h), HDR, 1, sampled), + bloom_a: texture("scene bloom", half, HDR, 1, sampled), + bloom_b: texture("scene bloom", half, HDR, 1, sampled), + output: gpu.target(w, h, "scene output"), + dummy_depth: lit_dummy_depth(device), + staging: None, + } + } +} + /// Render `scene` on the GPU for a canvas `width` by `height` logical pixels -/// at `pixel_scale`. `None` without a GPU (or geometry), so the caller can -/// fall back to the software renderer. +/// at `pixel_scale`, reusing `state`'s targets (made or remade as needed). +/// `None` without a GPU (or geometry), so the caller can fall back to the +/// software renderer. pub fn render_scene( scene: &Scene, camera: Camera, @@ -1067,6 +1128,7 @@ pub fn render_scene( height: f32, pixel_scale: f32, rgba: &dyn Fn(Color) -> [f32; 4], + state: &mut Option, ) -> Option { let gpu = gpu()?; let frame = scene.camera_frame(camera, width, height)?; @@ -1171,40 +1233,25 @@ pub fn render_scene( usage: wgpu::BufferUsages::UNIFORM, }); - // Targets. - use wgpu::TextureUsages as U; - let texture = |label: &str, size: (u32, u32), format, samples, usage| { - device.create_texture(&wgpu::TextureDescriptor { - label: Some(label), - size: wgpu::Extent3d { - width: size.0, - height: size.1, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: samples, - dimension: wgpu::TextureDimension::D2, - format, - usage, - view_formats: &[], - }) - }; - let sampled = U::RENDER_ATTACHMENT | U::TEXTURE_BINDING; - let shadow_map = texture( - "scene shadow map", - (SHADOW_SIZE, SHADOW_SIZE), - DEPTH, - 1, - sampled, - ); - let msaa_color = texture("scene color", (w, h), HDR, SAMPLES, U::RENDER_ATTACHMENT); - let msaa_depth = texture("scene depth", (w, h), DEPTH, SAMPLES, sampled); - let opaque_resolved = texture("scene behind glass", (w, h), HDR, 1, sampled); - let lit = texture("scene lit", (w, h), HDR, 1, sampled); + // Targets, kept between frames. + if state.as_ref().is_none_or(|t| t.size != (w, h)) { + *state = Some(SceneState::new(gpu, (w, h))); + } + let targets = state.as_mut().expect("made above"); + let SceneState { + shadow_map, + msaa_color, + msaa_depth, + opaque_resolved, + lit, + bloom_a, + bloom_b, + output, + dummy_depth, + staging, + .. + } = targets; let half = ((w / 2).max(1), (h / 2).max(1)); - let bloom_a = texture("scene bloom", half, HDR, 1, sampled); - let bloom_b = texture("scene bloom", half, HDR, 1, sampled); - let output = gpu.target(w, h, "scene output"); let view = |t: &wgpu::Texture| t.create_view(&Default::default()); let env_view = environment @@ -1247,7 +1294,7 @@ pub fn render_scene( }, wgpu::BindGroupEntry { binding: 1, - resource: wgpu::BindingResource::TextureView(&view(&shadow_map)), + resource: wgpu::BindingResource::TextureView(&view(shadow_map)), }, wgpu::BindGroupEntry { binding: 2, @@ -1307,7 +1354,7 @@ pub fn render_scene( }, wgpu::BindGroupEntry { binding: 1, - resource: wgpu::BindingResource::TextureView(&view(&lit_dummy_depth(device))), + resource: wgpu::BindingResource::TextureView(&view(dummy_depth)), }, wgpu::BindGroupEntry { binding: 2, @@ -1324,7 +1371,7 @@ pub fn render_scene( ], }); { - let shadow_view = view(&shadow_map); + let shadow_view = view(shadow_map); let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("scene shadow"), color_attachments: &[], @@ -1344,12 +1391,12 @@ pub fn render_scene( pass.set_bind_group(0, &shadow_globals, &[]); draw(&mut pass, &opaque, geometry.shadowed); } - let color_view = view(&msaa_color); - let depth_view = view(&msaa_depth); + let color_view = view(msaa_color); + let depth_view = view(msaa_depth); // 2. Opaque surfaces; see-through ones and sprites over them (so glass // refracts them too); then lines and points. { - let resolve = view(&opaque_resolved); + let resolve = view(opaque_resolved); let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("scene main"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -1397,7 +1444,7 @@ pub fn render_scene( entries: &[ wgpu::BindGroupEntry { binding: 0, - resource: wgpu::BindingResource::TextureView(&view(&opaque_resolved)), + resource: wgpu::BindingResource::TextureView(&view(opaque_resolved)), }, wgpu::BindGroupEntry { binding: 1, @@ -1405,7 +1452,7 @@ pub fn render_scene( }, ], }); - let resolve = view(&lit); + let resolve = view(lit); let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("scene glass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -1508,32 +1555,32 @@ pub fn render_scene( let step_y = post_uniform([0.0, 1.0 / half.1 as f32, 0.0, 0.0]); fullscreen( &mut encoder, - &bloom_a, + bloom_a, &pipelines.bright, - &post_bind(shaded, &zero, &bloom_b), + &post_bind(shaded, &zero, bloom_b), ); for _ in 0..2 { fullscreen( &mut encoder, - &bloom_b, + bloom_b, &pipelines.blur, - &post_bind(&bloom_a, &step_x, &lit_dummy(device)), + &post_bind(bloom_a, &step_x, &lit_dummy(device)), ); fullscreen( &mut encoder, - &bloom_a, + bloom_a, &pipelines.blur, - &post_bind(&bloom_b, &step_y, &lit_dummy(device)), + &post_bind(bloom_b, &step_y, &lit_dummy(device)), ); } let strength = post_uniform([0.9, 0.0, 0.0, 0.0]); fullscreen( &mut encoder, - &output, + output, &pipelines.composite, - &post_bind(shaded, &strength, &bloom_a), + &post_bind(shaded, &strength, bloom_a), ); - Some(gpu.read_back(encoder, &output, false)) + Some(gpu.read_back(encoder, output, staging)) } /// A 1×1 texture for bind slots a pass doesn't read (a texture can't be both @@ -1689,12 +1736,13 @@ mod tests { return; } let scene = gallery(); + let mut state = None; let camera = Camera { yaw: 0.5, pitch: 0.35, zoom: 1.0, }; - let frame = render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + let frame = render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette, &mut state).unwrap(); if let Ok(path) = std::env::var("ZERON_VISUAL_DUMP_ASSETS") { crate::dump_ppm(&frame, &path); } @@ -1716,7 +1764,7 @@ mod tests { .count(); assert!(blueish > 500, "{blueish}"); // A texture uploads once, however many frames draw it. - render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette, &mut state).unwrap(); } #[test] @@ -1725,15 +1773,16 @@ mod tests { return; } let scene = room(); + let mut state = None; let camera = Camera { yaw: 0.6, pitch: 0.45, zoom: 1.0, }; // The first frame also makes the device and compiles pipelines. - render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette, &mut state).unwrap(); let start = std::time::Instant::now(); - let frame = render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette).unwrap(); + let frame = render_scene(&scene, camera, 720.0, 360.0, 2.0, &palette, &mut state).unwrap(); eprintln!( "gpu scene {}x{} in {:?}", frame.width, @@ -1780,7 +1829,16 @@ mod tests { {shell} end }}"# ); - render_scene(&scene(&script), camera, 720.0, 360.0, 1.0, &palette).unwrap() + render_scene( + &scene(&script), + camera, + 720.0, + 360.0, + 1.0, + &palette, + &mut None, + ) + .unwrap() }; let bare = red_center(&render("")); // A faded shell around the box (like a cutaway wall) must not shift diff --git a/crates/visual-gpu/src/shader.rs b/crates/visual-gpu/src/shader.rs index 9a94ba9885..75542e2837 100644 --- a/crates/visual-gpu/src/shader.rs +++ b/crates/visual-gpu/src/shader.rs @@ -37,6 +37,7 @@ pub struct ShaderState { image: wgpu::Texture, bloom: [wgpu::Texture; 2], output: wgpu::Texture, + staging: Option, } pub(crate) struct Pipelines { @@ -401,6 +402,7 @@ impl ShaderState { texture(device, "shader bloom", half, HDR), ], output: texture(device, "shader output", size, FORMAT), + staging: None, } } } @@ -662,7 +664,7 @@ pub fn render_shader( &pipelines.composite, &post_bind(&state.image, &composite, &glow), ); - let frame = gpu.read_back(encoder, &state.output, false); + let frame = gpu.read_back(encoder, &state.output, &mut state.staging); state.latest = write; Ok(frame) }