diff --git a/CLAUDE.md b/CLAUDE.md index c1e47094b..1ee96cb52 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ On Windows, build from **x64 Native Tools Command Prompt for VS 2022** to avoid ``` src/ # Desktop app — main binary, GPUI views, app coordinator -crates/ # Library crates (29 crates, see below) +crates/ # Library crates (30 crates, see below) mobile/ # Mobile app — React Native UI (mobile/rn) over the Rust core via uniffi (crates/okena-mobile-ffi) web/ # Web client (React + TypeScript + xterm.js) assets/ # Fonts, icons (assets/icons/*.svg referenced as icons/*.svg) @@ -42,6 +42,7 @@ Most logic lives in `crates/`. The `src/` modules are thin re-exports (`pub use | `okena-theme` | Theming system (built-in + custom themes) | | `okena-ui` | Design tokens, shared UI utilities | | `okena-files` | File search, file viewer, syntax highlighting | +| `okena-highlight` | syntect/tree-sitter syntax highlighting shared by the file viewer, diff viewer and markdown code blocks | | `okena-markdown` | Markdown parsing and rendering | | `okena-views-terminal` | Terminal pane, layout container, split/tabs views | | `okena-views-sidebar` | Sidebar, project list, folder list, drag-and-drop | diff --git a/Cargo.lock b/Cargo.lock index 2d172c7ce..fce060e44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6259,18 +6259,15 @@ dependencies = [ "log", "nucleo-matcher", "okena-core", + "okena-highlight", "okena-markdown", "okena-transport", "okena-ui", "serde", "serde_json", - "streaming-iterator", "syntect", "tempfile", - "tree-sitter", - "tree-sitter-md", "ttf-parser", - "two-face", "usvg 0.45.1", ] @@ -6290,6 +6287,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "okena-highlight" +version = "0.1.0" +dependencies = [ + "gpui", + "streaming-iterator", + "syntect", + "tree-sitter", + "tree-sitter-md", + "two-face", +] + [[package]] name = "okena-hooks" version = "0.1.0" @@ -6322,6 +6331,7 @@ dependencies = [ "gpui", "gpui-component", "okena-core", + "okena-highlight", "okena-ui", "pulldown-cmark", "serde_yaml_ng", diff --git a/Cargo.toml b/Cargo.toml index dbcb2fa06..eb1333fc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/okena-mobile-ffi", "crates/okena-core", "crates/okena-transport", "crates/okena-git", "crates/okena-views-git", "crates/okena-views-services", "crates/okena-views-sidebar", "crates/okena-views-terminal", "crates/okena-terminal", "crates/okena-layout", "crates/okena-state", "crates/okena-hooks", "crates/okena-workspace", "crates/okena-ui", "crates/okena-usage", "crates/okena-files", "crates/okena-markdown", "crates/okena-extensions", "crates/okena-ext-claude", "crates/okena-ext-codex", "crates/okena-ext-github", "crates/okena-ext-updater", "crates/okena-services", "crates/okena-remote-client", "crates/okena-remote-server", "crates/okena-views-remote", "crates/okena-theme", "crates/okena-cli", "crates/okena-app-core", "crates/okena-app", "crates/okena-daemon-core", "crates/okena-daemon", "crates/okena-tui"] +members = [".", "crates/okena-mobile-ffi", "crates/okena-core", "crates/okena-transport", "crates/okena-git", "crates/okena-views-git", "crates/okena-views-services", "crates/okena-views-sidebar", "crates/okena-views-terminal", "crates/okena-terminal", "crates/okena-layout", "crates/okena-state", "crates/okena-hooks", "crates/okena-workspace", "crates/okena-ui", "crates/okena-usage", "crates/okena-files", "crates/okena-markdown", "crates/okena-highlight", "crates/okena-extensions", "crates/okena-ext-claude", "crates/okena-ext-codex", "crates/okena-ext-github", "crates/okena-ext-updater", "crates/okena-services", "crates/okena-remote-client", "crates/okena-remote-server", "crates/okena-views-remote", "crates/okena-theme", "crates/okena-cli", "crates/okena-app-core", "crates/okena-app", "crates/okena-daemon-core", "crates/okena-daemon", "crates/okena-tui"] resolver = "2" [workspace.package] diff --git a/crates/okena-files/Cargo.toml b/crates/okena-files/Cargo.toml index e2417bd3a..94808a5f7 100644 --- a/crates/okena-files/Cargo.toml +++ b/crates/okena-files/Cargo.toml @@ -11,11 +11,8 @@ gpui = [ "dep:gpui-component", "dep:okena-ui", "dep:okena-markdown", + "dep:okena-highlight", "dep:syntect", - "dep:two-face", - "dep:tree-sitter", - "dep:tree-sitter-md", - "dep:streaming-iterator", "dep:usvg", "dep:ttf-parser", "dep:image", @@ -25,20 +22,15 @@ gpui = [ okena-core = { path = "../okena-core" } okena-transport = { path = "../okena-transport", features = ["cancellable-http"] } okena-markdown = { path = "../okena-markdown", optional = true } +okena-highlight = { path = "../okena-highlight", optional = true } okena-ui = { path = "../okena-ui", optional = true } gpui = { git = "https://github.com/zed-industries/zed", package = "gpui", optional = true } gpui-component = { git = "https://github.com/longbridge/gpui-component", package = "gpui-component", optional = true } +# The highlighting itself lives in okena-highlight; syntect stays here for the +# `SyntaxSet` the viewers hold and pass back in. syntect = { version = "5", features = ["default-fancy"], optional = true } -two-face = { version = "0.5", default-features = false, features = ["syntect-fancy"], optional = true } - -# Markdown highlighting: tree-sitter is ~20x faster than syntect's Markdown -# grammar on large files. tree-sitter (0.26) and streaming-iterator are already -# in the workspace via gpui-component, so this only adds the grammar crate. -tree-sitter = { version = "0.26", optional = true } -tree-sitter-md = { version = "0.5", features = ["parser"], optional = true } -streaming-iterator = { version = "0.1", optional = true } grep-searcher = "0.1" grep-regex = "0.1" diff --git a/crates/okena-files/src/code_view.rs b/crates/okena-files/src/code_view.rs index 5cfa6df24..5b3198e93 100644 --- a/crates/okena-files/src/code_view.rs +++ b/crates/okena-files/src/code_view.rs @@ -6,7 +6,7 @@ //! - Text selection utilities use crate::selection::SelectionState; -use crate::syntax::{HighlightedLine, HighlightedSpan}; +use crate::syntax::HighlightedLine; use gpui::*; /// Type alias for code selection (line index, column). @@ -93,89 +93,10 @@ pub fn update_scrollbar_drag( .set_offset(point(px(0.0), px(-new_scroll))); } -/// Build a StyledText with optional background highlights (e.g. selection or word-level diff). -/// Splits syntax color highlights at background range boundaries to produce -/// non-overlapping highlights (required by `StyledText::compute_runs`). -pub fn build_styled_text_with_backgrounds( - spans: &[HighlightedSpan], - bg_ranges: &[(std::ops::Range, Hsla)], -) -> StyledText { - let mut text = String::new(); - let mut highlights = Vec::new(); - - for span in spans { - text.push_str(&span.text); - } - - if bg_ranges.is_empty() { - // Fast path: no background highlights, just syntax colors - let mut offset = 0; - for span in spans { - let start = offset; - offset += span.text.len(); - if start < offset { - highlights.push(( - start..offset, - HighlightStyle { - color: Some(span.color.into()), - ..Default::default() - }, - )); - } - } - } else { - // Split syntax spans at background range boundaries so no highlights overlap - let mut offset = 0; - for span in spans { - let span_start = offset; - let span_end = offset + span.text.len(); - offset = span_end; - - if span_start >= span_end { - continue; - } - - // Collect boundary points from bg_ranges that fall within this span - let mut boundaries = vec![span_start]; - for (br, _) in bg_ranges { - if br.start > span_start && br.start < span_end { - boundaries.push(br.start); - } - if br.end > span_start && br.end < span_end { - boundaries.push(br.end); - } - } - boundaries.push(span_end); - boundaries.sort(); - boundaries.dedup(); - - for window in boundaries.windows(2) { - let sub_start = window[0]; - let sub_end = window[1]; - if sub_start >= sub_end { - continue; - } - - let mut style = HighlightStyle { - color: Some(span.color.into()), - ..Default::default() - }; - - // Apply background if this sub-range falls within any background range - for (br, bg_color) in bg_ranges { - if sub_start >= br.start && sub_end <= br.end { - style.background_color = Some(*bg_color); - break; - } - } - - highlights.push((sub_start..sub_end, style)); - } - } - } - - StyledText::new(text).with_highlights(highlights) -} +// Lives in `okena-highlight` (shared with the markdown renderer); re-exported so +// the code viewers keep their `code_view::build_styled_text_with_backgrounds` +// import. +pub use okena_highlight::styled::build_styled_text_with_backgrounds; /// Compute selection background ranges for a single line. /// diff --git a/crates/okena-files/src/file_viewer/loading.rs b/crates/okena-files/src/file_viewer/loading.rs index 7789cec86..b0dd39c54 100644 --- a/crates/okena-files/src/file_viewer/loading.rs +++ b/crates/okena-files/src/file_viewer/loading.rs @@ -256,7 +256,9 @@ impl FileViewerTab { self.content = content; self.do_highlight_content(&self.file_path.clone(), syntax_set, is_dark); if self.is_markdown { - self.markdown_doc = Some(MarkdownDocument::parse(&self.content)); + let mut doc = MarkdownDocument::parse(&self.content); + doc.highlight_code_blocks(is_dark); + self.markdown_doc = Some(doc); } } Ok(LoadedContent::Image { decoded, source }) => { diff --git a/crates/okena-files/src/file_viewer/mod.rs b/crates/okena-files/src/file_viewer/mod.rs index abc415ed6..08a9a1213 100644 --- a/crates/okena-files/src/file_viewer/mod.rs +++ b/crates/okena-files/src/file_viewer/mod.rs @@ -980,6 +980,11 @@ impl FileViewer { // source-view XML), so they need the rehighlight too. if rehighlight && !tab.is_font && (!tab.is_image || tab.is_svg) { tab.do_highlight_content(&tab.file_path.clone(), &self.syntax_set, self.is_dark); + // The rendered markdown view carries its own highlighted code + // blocks, separate from the source view's lines. + if let Some(doc) = tab.markdown_doc.as_mut() { + doc.highlight_code_blocks(self.is_dark); + } } } } diff --git a/crates/okena-files/src/lib.rs b/crates/okena-files/src/lib.rs index 7b784064f..c22e97405 100644 --- a/crates/okena-files/src/lib.rs +++ b/crates/okena-files/src/lib.rs @@ -24,11 +24,11 @@ pub mod in_page_search; #[cfg(feature = "gpui")] pub mod list_overlay; #[cfg(feature = "gpui")] -pub mod markdown_highlight; -#[cfg(feature = "gpui")] pub mod selection; +// Highlighting lives in `okena-highlight` (shared with the markdown renderer); +// re-exported here so the viewers keep their `okena_files::syntax` paths. #[cfg(feature = "gpui")] -pub mod syntax; +pub use okena_highlight::{markdown_highlight, syntax}; // `theme` re-exports gpui theme helpers from `okena-ui` (a gpui crate) and is // consumed only by gpui code, so it is gated with the rest of the viewer even // though it holds no rendering itself. diff --git a/crates/okena-highlight/Cargo.toml b/crates/okena-highlight/Cargo.toml new file mode 100644 index 000000000..5bc790533 --- /dev/null +++ b/crates/okena-highlight/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "okena-highlight" +version = "0.1.0" +edition = "2024" +license = "MIT" + +[dependencies] +gpui = { git = "https://github.com/zed-industries/zed", package = "gpui" } + +syntect = { version = "5", features = ["default-fancy"] } +two-face = { version = "0.5", default-features = false, features = ["syntect-fancy"] } + +# Markdown highlighting: tree-sitter is ~20x faster than syntect's Markdown +# grammar on large files. tree-sitter (0.26) and streaming-iterator are already +# in the workspace via gpui-component, so this only adds the grammar crate. +tree-sitter = "0.26" +tree-sitter-md = { version = "0.5", features = ["parser"] } +streaming-iterator = "0.1" diff --git a/crates/okena-highlight/src/lib.rs b/crates/okena-highlight/src/lib.rs new file mode 100644 index 000000000..949bafbd1 --- /dev/null +++ b/crates/okena-highlight/src/lib.rs @@ -0,0 +1,11 @@ +#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))] + +//! Syntax highlighting shared by every viewer that shows code: the file viewer, +//! the diff viewer, and markdown code blocks. +//! +//! It sits below `okena-files` and `okena-markdown` so all three can share one +//! `SyntaxSet` — loading it is expensive and a second copy would be megabytes. + +pub mod markdown_highlight; +pub mod styled; +pub mod syntax; diff --git a/crates/okena-files/src/markdown_highlight.rs b/crates/okena-highlight/src/markdown_highlight.rs similarity index 97% rename from crates/okena-files/src/markdown_highlight.rs rename to crates/okena-highlight/src/markdown_highlight.rs index db546ec79..fe09c1130 100644 --- a/crates/okena-files/src/markdown_highlight.rs +++ b/crates/okena-highlight/src/markdown_highlight.rs @@ -19,7 +19,7 @@ use crate::syntax::{ HighlightedLine, HighlightedSpan, default_text_color, highlight_line, load_syntax_theme, - map_extension_to_syntax, + syntax_for_language, }; use gpui::Rgba; use std::collections::HashMap; @@ -229,7 +229,7 @@ fn collect_code_blocks( let Some(code) = content.get(start..end.min(content.len())) else { continue; }; - let syntax = syntax_for_lang(lang, syntax_set); + let syntax = syntax_for_language(lang, syntax_set); let mut highlighter = HighlightLines::new(syntax, theme); let first_line = line_of(line_starts, start); for (i, line) in LinesWithEndings::from(code).enumerate() { @@ -239,18 +239,6 @@ fn collect_code_blocks( } } -/// Resolve a fenced-code info-string language to a syntect syntax, falling back -/// to plain text. Mirrors [`crate::syntax::get_syntax_for_path`]'s mapping. -fn syntax_for_lang<'a>( - lang: &str, - syntax_set: &'a SyntaxSet, -) -> &'a syntect::parsing::SyntaxReference { - map_extension_to_syntax(lang) - .and_then(|mapped| syntax_set.find_syntax_by_extension(mapped)) - .or_else(|| syntax_set.find_syntax_by_token(lang)) - .unwrap_or_else(|| syntax_set.find_syntax_plain_text()) -} - /// Build spans for one (non-code) line from the per-byte colour buffer, /// coalescing equal-coloured runs, expanding tabs, and dropping line endings. fn line_spans( diff --git a/crates/okena-highlight/src/styled.rs b/crates/okena-highlight/src/styled.rs new file mode 100644 index 000000000..d05b06313 --- /dev/null +++ b/crates/okena-highlight/src/styled.rs @@ -0,0 +1,91 @@ +//! Turning highlighted spans into a GPUI `StyledText`. +//! +//! Shared by every code surface: the file viewer, the diff viewer, and markdown +//! code blocks all colour the same span data the same way. + +use crate::syntax::HighlightedSpan; +use gpui::*; + +/// Build a StyledText with optional background highlights (e.g. selection or word-level diff). +/// Splits syntax color highlights at background range boundaries to produce +/// non-overlapping highlights (required by `StyledText::compute_runs`). +pub fn build_styled_text_with_backgrounds( + spans: &[HighlightedSpan], + bg_ranges: &[(std::ops::Range, Hsla)], +) -> StyledText { + let mut text = String::new(); + let mut highlights = Vec::new(); + + for span in spans { + text.push_str(&span.text); + } + + if bg_ranges.is_empty() { + // Fast path: no background highlights, just syntax colors + let mut offset = 0; + for span in spans { + let start = offset; + offset += span.text.len(); + if start < offset { + highlights.push(( + start..offset, + HighlightStyle { + color: Some(span.color.into()), + ..Default::default() + }, + )); + } + } + } else { + // Split syntax spans at background range boundaries so no highlights overlap + let mut offset = 0; + for span in spans { + let span_start = offset; + let span_end = offset + span.text.len(); + offset = span_end; + + if span_start >= span_end { + continue; + } + + // Collect boundary points from bg_ranges that fall within this span + let mut boundaries = vec![span_start]; + for (br, _) in bg_ranges { + if br.start > span_start && br.start < span_end { + boundaries.push(br.start); + } + if br.end > span_start && br.end < span_end { + boundaries.push(br.end); + } + } + boundaries.push(span_end); + boundaries.sort(); + boundaries.dedup(); + + for window in boundaries.windows(2) { + let sub_start = window[0]; + let sub_end = window[1]; + if sub_start >= sub_end { + continue; + } + + let mut style = HighlightStyle { + color: Some(span.color.into()), + ..Default::default() + }; + + // Apply background if this sub-range falls within any background range + for (br, bg_color) in bg_ranges { + if sub_start >= br.start && sub_end <= br.end { + style.background_color = Some(*bg_color); + break; + } + } + + highlights.push((sub_start..sub_end, style)); + } + } + } + + StyledText::new(text).with_highlights(highlights) +} diff --git a/crates/okena-files/src/syntax.rs b/crates/okena-highlight/src/syntax.rs similarity index 78% rename from crates/okena-files/src/syntax.rs rename to crates/okena-highlight/src/syntax.rs index ed6ae8215..8fcbbae2c 100644 --- a/crates/okena-files/src/syntax.rs +++ b/crates/okena-highlight/src/syntax.rs @@ -1,14 +1,14 @@ //! Shared syntax highlighting utilities. //! //! Provides types and functions for syntax highlighting that can be used -//! across different viewers (file viewer, diff viewer, etc.). +//! across different viewers (file viewer, diff viewer, markdown code blocks). use gpui::Rgba; use std::path::Path; use std::sync::{Arc, OnceLock}; use syntect::easy::HighlightLines; use syntect::highlighting::Theme; -use syntect::parsing::SyntaxSet; +use syntect::parsing::{SyntaxReference, SyntaxSet}; use syntect::util::LinesWithEndings; /// Global cached syntax set with extended syntaxes (including TypeScript/TSX). @@ -135,10 +135,7 @@ pub fn map_extension_to_syntax(ext: &str) -> Option<&'static str> { } /// Get syntax reference for a file path. -pub fn get_syntax_for_path<'a>( - path: &Path, - syntax_set: &'a SyntaxSet, -) -> &'a syntect::parsing::SyntaxReference { +pub fn get_syntax_for_path<'a>(path: &Path, syntax_set: &'a SyntaxSet) -> &'a SyntaxReference { let ext = path.extension().and_then(|e| e.to_str()); ext.and_then(map_extension_to_syntax) @@ -213,6 +210,48 @@ pub fn highlight_line( } } +/// Resolve a fenced-code info string (`rust`, `ts`, `sh`) to a syntect syntax. +/// `None` when the language is unknown, so callers can tell "highlight as plain +/// text" apart from "leave this block alone". Mirrors [`get_syntax_for_path`]'s +/// mapping. +pub fn find_syntax_for_language<'a>( + lang: &str, + syntax_set: &'a SyntaxSet, +) -> Option<&'a SyntaxReference> { + map_extension_to_syntax(lang) + .and_then(|mapped| syntax_set.find_syntax_by_extension(mapped)) + .or_else(|| syntax_set.find_syntax_by_token(lang)) +} + +/// Like [`find_syntax_for_language`], falling back to plain text. +pub fn syntax_for_language<'a>(lang: &str, syntax_set: &'a SyntaxSet) -> &'a SyntaxReference { + find_syntax_for_language(lang, syntax_set) + .unwrap_or_else(|| syntax_set.find_syntax_plain_text()) +} + +/// Highlight a fenced markdown code block, one entry per line of `code`. +/// +/// A fence with no language, or one syntect does not know, comes back empty: +/// plain-text highlighting would repaint the block in the syntax theme's +/// foreground, and a code block with nothing to colour should keep the +/// document's own text colour instead. +/// +/// Unlike [`highlight_content`] this loads the shared `SyntaxSet` itself — the +/// markdown renderer has no file open and so holds no set of its own. +pub fn highlight_code_block( + code: &str, + language: Option<&str>, + is_dark: bool, +) -> Vec { + let syntax_set = load_syntax_set(); + let Some(syntax) = language.and_then(|lang| find_syntax_for_language(lang, &syntax_set)) else { + return Vec::new(); + }; + // Tabs stay as-is: the markdown renderer maps selection offsets onto the + // raw code text, so a span must keep the character count of its line. + highlight_lines(code, syntax, &syntax_set, 0, is_dark, false) +} + /// Highlight file content and return a vector of highlighted lines. /// /// # Arguments @@ -236,7 +275,35 @@ pub fn highlight_content( ); } - let syntax = get_syntax_for_path(path, syntax_set); + highlight_lines( + content, + get_syntax_for_path(path, syntax_set), + syntax_set, + max_lines, + is_dark, + true, + ) +} + +/// Highlight `content` line by line, merging adjacent spans of the same colour. +/// `expand_tabs` turns each tab into four spaces — right for a viewer that owns +/// its own layout, wrong where the caller indexes back into the original text. +fn highlight_lines( + content: &str, + syntax: &SyntaxReference, + syntax_set: &SyntaxSet, + max_lines: usize, + is_dark: bool, + expand_tabs: bool, +) -> Vec { + let expand = |text: &str| { + if expand_tabs { + text.replace('\t', " ") + } else { + text.to_string() + } + }; + let theme = load_syntax_theme(is_dark); let mut highlighter = HighlightLines::new(syntax, theme); let default_color = default_text_color(is_dark); @@ -263,9 +330,7 @@ pub fn highlight_content( }; // Pre-process text: remove newlines, expand tabs - let processed = text - .trim_end_matches(&['\n', '\r'][..]) - .replace('\t', " "); + let processed = expand(text.trim_end_matches(&['\n', '\r'][..])); if processed.is_empty() { continue; @@ -292,9 +357,7 @@ pub fn highlight_content( (merged, plain) } Err(_) => { - let text = line - .trim_end_matches(&['\n', '\r'][..]) - .replace('\t', " "); + let text = expand(line.trim_end_matches(&['\n', '\r'][..])); ( vec![HighlightedSpan { color: default_color, diff --git a/crates/okena-markdown/Cargo.toml b/crates/okena-markdown/Cargo.toml index 70be2aa03..bfa001f12 100644 --- a/crates/okena-markdown/Cargo.toml +++ b/crates/okena-markdown/Cargo.toml @@ -6,6 +6,7 @@ license = "MIT" [dependencies] okena-core = { path = "../okena-core" } +okena-highlight = { path = "../okena-highlight" } okena-ui = { path = "../okena-ui" } gpui = { git = "https://github.com/zed-industries/zed", package = "gpui" } diff --git a/crates/okena-markdown/src/lib.rs b/crates/okena-markdown/src/lib.rs index 7ab7ead25..e75f16bb5 100644 --- a/crates/okena-markdown/src/lib.rs +++ b/crates/okena-markdown/src/lib.rs @@ -51,3 +51,83 @@ pub struct MarkdownDocument { /// Flat text representation of all visible content pub plain_text: String, } + +impl MarkdownDocument { + /// Syntax-highlight every fenced code block for the given theme. + /// + /// Kept out of `parse` because the colours come from the syntax theme, which + /// the parser has no business knowing: the viewer calls this after parsing + /// and again whenever the theme flips between dark and light. A document + /// that never gets the call renders its code blocks in the document text + /// colour, exactly as before. + pub fn highlight_code_blocks(&mut self, is_dark: bool) { + for node in &mut self.nodes { + if let Node::CodeBlock { + language, + code, + highlighted, + } = node + { + *highlighted = okena_highlight::syntax::highlight_code_block( + code, + language.as_deref(), + is_dark, + ) + .into_iter() + .map(|line| line.spans) + .collect(); + } + } + } +} + +#[cfg(test)] +mod tests { + // Named imports, not a glob: `use super::*` would pull in gpui's own `test` + // macro and shadow the one these tests need. + use super::{MarkdownDocument, Node}; + + /// The spans of a line must hold exactly that line's characters — tabs and + /// all. Rendering maps a character-offset selection onto them, so a span set + /// that drops or rewrites characters silently shifts the selection. + #[test] + fn spans_reproduce_each_line_verbatim() { + let mut doc = MarkdownDocument::parse("```rust\nfn main() {\n\tlet x = 1;\n}\n```\n"); + doc.highlight_code_blocks(true); + + let Some(Node::CodeBlock { + code, highlighted, .. + }) = doc.nodes.first() + else { + panic!("expected a code block"); + }; + + let lines: Vec<&str> = code.lines().collect(); + assert_eq!(highlighted.len(), lines.len()); + for (spans, line) in highlighted.iter().zip(&lines) { + let joined: String = spans.iter().map(|s| s.text.as_str()).collect(); + assert_eq!(&joined, line); + } + // More than one colour, or nothing was actually highlighted. + let colors: Vec<_> = highlighted[0].iter().map(|s| s.color.r).collect(); + assert!(colors.len() > 1, "expected `fn main() {{` to be coloured"); + } + + /// A fence with no language, or one syntect cannot place, is left alone so + /// it keeps the document's text colour instead of the syntax theme's. + #[test] + fn unknown_and_missing_languages_stay_unhighlighted() { + for md in [ + "```\nplain text\n```\n", + "```notalanguage\nplain text\n```\n", + ] { + let mut doc = MarkdownDocument::parse(md); + doc.highlight_code_blocks(true); + + let Some(Node::CodeBlock { highlighted, .. }) = doc.nodes.first() else { + panic!("expected a code block"); + }; + assert!(highlighted.is_empty(), "{md:?} should not be highlighted"); + } + } +} diff --git a/crates/okena-markdown/src/parser.rs b/crates/okena-markdown/src/parser.rs index 9ec553d58..c2e27b540 100644 --- a/crates/okena-markdown/src/parser.rs +++ b/crates/okena-markdown/src/parser.rs @@ -122,6 +122,7 @@ impl MarkdownDocument { nodes.push(Node::CodeBlock { language: code_block_lang.take(), code: std::mem::take(&mut code_block_content), + highlighted: Vec::new(), }); in_code_block = false; } diff --git a/crates/okena-markdown/src/render.rs b/crates/okena-markdown/src/render.rs index 3c2336578..6c5f7666b 100644 --- a/crates/okena-markdown/src/render.rs +++ b/crates/okena-markdown/src/render.rs @@ -4,6 +4,8 @@ use gpui::prelude::FluentBuilder; use gpui::*; use gpui_component::{h_flex, v_flex}; use okena_core::theme::ThemeColors; +use okena_highlight::styled::build_styled_text_with_backgrounds; +use okena_highlight::syntax::HighlightedSpan; use okena_ui::code_block::code_block_container; use okena_ui::tokens::ui_text_md; @@ -18,6 +20,33 @@ use super::{MarkdownDocument, RenderedNode}; /// its own selectable element), so this stands in for `line_height`. const CODE_LINE_HEIGHT: Pixels = px(20.0); +/// One syntax-highlighted code line, drawn as a single text run so indentation +/// and wide glyphs measure the same as the source. +/// +/// `selection` is a character range within the line; the spans hold the line's +/// characters unchanged (tabs included), so the range converts straight to the +/// byte offsets `build_styled_text_with_backgrounds` expects. +fn highlighted_code_line( + line: &str, + spans: &[HighlightedSpan], + selection: Option<(usize, usize)>, + selection_bg: Rgba, +) -> StyledText { + let byte_at = |char_idx: usize| { + line.char_indices() + .nth(char_idx) + .map(|(byte, _)| byte) + .unwrap_or(line.len()) + }; + let bg_ranges = match selection { + Some((start, end)) if start < end => { + vec![(byte_at(start)..byte_at(end), selection_bg.into())] + } + _ => Vec::new(), + }; + build_styled_text_with_backgrounds(spans, &bg_ranges) +} + impl MarkdownDocument { /// Number of top-level blocks in the document. Each maps to one list item. pub fn node_count(&self) -> usize { @@ -55,13 +84,17 @@ impl MarkdownDocument { }); let rendered = match node { - Node::CodeBlock { language, code } => { + Node::CodeBlock { + language, + code, + highlighted, + } => { // Return code blocks with individual lines for per-line selection let selection_bg = rgba(0x3390ff40); let mut lines = Vec::new(); let mut line_offset = offset; - for line in code.lines() { + for (line_idx, line) in code.lines().enumerate() { let line_len = char_len(line); let line_end = line_offset + line_len + 1; // +1 for newline @@ -75,7 +108,15 @@ impl MarkdownDocument { } }); - let line_div = if let Some((sel_start, sel_end)) = line_sel { + let spans = highlighted.get(line_idx).map(Vec::as_slice).unwrap_or(&[]); + let line_div = if !spans.is_empty() { + div().h(CODE_LINE_HEIGHT).child(highlighted_code_line( + line, + spans, + line_sel, + selection_bg, + )) + } else if let Some((sel_start, sel_end)) = line_sel { let (before, selected, after) = slice_by_chars(line, sel_start, sel_end); div() .h(CODE_LINE_HEIGHT) @@ -341,14 +382,18 @@ impl MarkdownDocument { Node::Paragraph { children } => { Self::render_inlines_with_selection(children, t, cx, selection).w_full() } - Node::CodeBlock { language, code } => { + Node::CodeBlock { + language, + code, + highlighted, + } => { let selection_bg = rgba(0x3390ff40); // Render code lines with selection let mut code_lines: Vec
= Vec::new(); let mut offset = 0usize; - for line in code.lines() { + for (line_idx, line) in code.lines().enumerate() { let line_len = char_len(line); let line_end = offset + line_len + 1; // +1 for newline @@ -360,7 +405,15 @@ impl MarkdownDocument { } }); - let line_div = if let Some((sel_start, sel_end)) = line_sel { + let spans = highlighted.get(line_idx).map(Vec::as_slice).unwrap_or(&[]); + let line_div = if !spans.is_empty() { + div().h(CODE_LINE_HEIGHT).child(highlighted_code_line( + line, + spans, + line_sel, + selection_bg, + )) + } else if let Some((sel_start, sel_end)) = line_sel { let (before, selected, after) = slice_by_chars(line, sel_start, sel_end); div() .h(CODE_LINE_HEIGHT) diff --git a/crates/okena-markdown/src/types.rs b/crates/okena-markdown/src/types.rs index 603f769e5..3b5f53cba 100644 --- a/crates/okena-markdown/src/types.rs +++ b/crates/okena-markdown/src/types.rs @@ -1,5 +1,7 @@ //! AST types and utility functions for the markdown renderer. +use okena_highlight::syntax::HighlightedSpan; + /// A node in the markdown AST. #[derive(Clone)] pub(crate) enum Node { @@ -13,6 +15,12 @@ pub(crate) enum Node { CodeBlock { language: Option, code: String, + /// Syntax-highlighted spans, one entry per line of `code`. Empty until + /// `MarkdownDocument::highlight_code_blocks` runs — it needs the theme, + /// which parsing does not have — and for languages syntect can't place. + /// Each line's spans hold exactly the characters of that line, so + /// selection offsets map onto them unchanged. + highlighted: Vec>, }, List { ordered: bool,