From 312fce2e0f53901057b2fd3462936318e7bd6773 Mon Sep 17 00:00:00 2001 From: David Matejka Date: Tue, 18 Aug 2026 16:08:55 +0200 Subject: [PATCH 1/2] refactor(highlight): extract highlighting into its own crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `okena-files` owned the syntect setup — the shared `SyntaxSet`, the theme pair, the per-line span builder — and `okena-views-git` reached into it for the diff viewer. That works while only viewers need colour, but the markdown renderer needs it too, and `okena-files` already depends on `okena-markdown`, so the dependency cannot run that way. `okena-highlight` now sits below both. `syntax.rs` and `markdown_highlight.rs` move across unchanged; `build_styled_text_with_backgrounds` moves out of `code_view` for the same reason. `okena-files` re-exports all three, so every `okena_files::syntax::…` and `code_view::build_styled_text_with_backgrounds` import keeps working. Loading the `SyntaxSet` twice would have cost megabytes for nothing, which is what a second copy in `okena-markdown` would have meant. The name says highlighting rather than syntax because #188 is adding an `okena-syntax` for something else — tree-sitter facts about code structure, no colours involved. `highlight_code_block` is the entry point a fenced block needs: it resolves a language token rather than a file path, and loads the `SyntaxSet` itself since the markdown renderer holds no file. It returns nothing for a fence with no language or one syntect cannot place — plain-text highlighting would repaint such a block in the syntax theme's foreground, and a block with nothing to colour should keep the document's own text colour. Tabs are left alone there: the caller maps character offsets onto the spans, so expanding a tab to four spaces would shift every selection on the line. The file path keeps expanding them, as it always has. `syntax_for_lang` in `markdown_highlight` was the same mapping as the new resolver and is now that resolver. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AWSp87PWoqnyB6prN3U3qD --- CLAUDE.md | 3 +- Cargo.lock | 17 +++- Cargo.toml | 2 +- crates/okena-files/Cargo.toml | 16 +--- crates/okena-files/src/code_view.rs | 89 +----------------- crates/okena-files/src/lib.rs | 6 +- crates/okena-highlight/Cargo.toml | 18 ++++ crates/okena-highlight/src/lib.rs | 11 +++ .../src/markdown_highlight.rs | 16 +--- crates/okena-highlight/src/styled.rs | 91 +++++++++++++++++++ .../src/syntax.rs | 89 +++++++++++++++--- 11 files changed, 226 insertions(+), 132 deletions(-) create mode 100644 crates/okena-highlight/Cargo.toml create mode 100644 crates/okena-highlight/src/lib.rs rename crates/{okena-files => okena-highlight}/src/markdown_highlight.rs (97%) create mode 100644 crates/okena-highlight/src/styled.rs rename crates/{okena-files => okena-highlight}/src/syntax.rs (78%) diff --git a/CLAUDE.md b/CLAUDE.md index c1e47094b..1ee96cb52 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ On Windows, build from **x64 Native Tools Command Prompt for VS 2022** to avoid ``` src/ # Desktop app — main binary, GPUI views, app coordinator -crates/ # Library crates (29 crates, see below) +crates/ # Library crates (30 crates, see below) mobile/ # Mobile app — React Native UI (mobile/rn) over the Rust core via uniffi (crates/okena-mobile-ffi) web/ # Web client (React + TypeScript + xterm.js) assets/ # Fonts, icons (assets/icons/*.svg referenced as icons/*.svg) @@ -42,6 +42,7 @@ Most logic lives in `crates/`. The `src/` modules are thin re-exports (`pub use | `okena-theme` | Theming system (built-in + custom themes) | | `okena-ui` | Design tokens, shared UI utilities | | `okena-files` | File search, file viewer, syntax highlighting | +| `okena-highlight` | syntect/tree-sitter syntax highlighting shared by the file viewer, diff viewer and markdown code blocks | | `okena-markdown` | Markdown parsing and rendering | | `okena-views-terminal` | Terminal pane, layout container, split/tabs views | | `okena-views-sidebar` | Sidebar, project list, folder list, drag-and-drop | diff --git a/Cargo.lock b/Cargo.lock index 2d172c7ce..7257a89cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6259,18 +6259,15 @@ dependencies = [ "log", "nucleo-matcher", "okena-core", + "okena-highlight", "okena-markdown", "okena-transport", "okena-ui", "serde", "serde_json", - "streaming-iterator", "syntect", "tempfile", - "tree-sitter", - "tree-sitter-md", "ttf-parser", - "two-face", "usvg 0.45.1", ] @@ -6290,6 +6287,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "okena-highlight" +version = "0.1.0" +dependencies = [ + "gpui", + "streaming-iterator", + "syntect", + "tree-sitter", + "tree-sitter-md", + "two-face", +] + [[package]] name = "okena-hooks" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index dbcb2fa06..eb1333fc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/okena-mobile-ffi", "crates/okena-core", "crates/okena-transport", "crates/okena-git", "crates/okena-views-git", "crates/okena-views-services", "crates/okena-views-sidebar", "crates/okena-views-terminal", "crates/okena-terminal", "crates/okena-layout", "crates/okena-state", "crates/okena-hooks", "crates/okena-workspace", "crates/okena-ui", "crates/okena-usage", "crates/okena-files", "crates/okena-markdown", "crates/okena-extensions", "crates/okena-ext-claude", "crates/okena-ext-codex", "crates/okena-ext-github", "crates/okena-ext-updater", "crates/okena-services", "crates/okena-remote-client", "crates/okena-remote-server", "crates/okena-views-remote", "crates/okena-theme", "crates/okena-cli", "crates/okena-app-core", "crates/okena-app", "crates/okena-daemon-core", "crates/okena-daemon", "crates/okena-tui"] +members = [".", "crates/okena-mobile-ffi", "crates/okena-core", "crates/okena-transport", "crates/okena-git", "crates/okena-views-git", "crates/okena-views-services", "crates/okena-views-sidebar", "crates/okena-views-terminal", "crates/okena-terminal", "crates/okena-layout", "crates/okena-state", "crates/okena-hooks", "crates/okena-workspace", "crates/okena-ui", "crates/okena-usage", "crates/okena-files", "crates/okena-markdown", "crates/okena-highlight", "crates/okena-extensions", "crates/okena-ext-claude", "crates/okena-ext-codex", "crates/okena-ext-github", "crates/okena-ext-updater", "crates/okena-services", "crates/okena-remote-client", "crates/okena-remote-server", "crates/okena-views-remote", "crates/okena-theme", "crates/okena-cli", "crates/okena-app-core", "crates/okena-app", "crates/okena-daemon-core", "crates/okena-daemon", "crates/okena-tui"] resolver = "2" [workspace.package] diff --git a/crates/okena-files/Cargo.toml b/crates/okena-files/Cargo.toml index e2417bd3a..94808a5f7 100644 --- a/crates/okena-files/Cargo.toml +++ b/crates/okena-files/Cargo.toml @@ -11,11 +11,8 @@ gpui = [ "dep:gpui-component", "dep:okena-ui", "dep:okena-markdown", + "dep:okena-highlight", "dep:syntect", - "dep:two-face", - "dep:tree-sitter", - "dep:tree-sitter-md", - "dep:streaming-iterator", "dep:usvg", "dep:ttf-parser", "dep:image", @@ -25,20 +22,15 @@ gpui = [ okena-core = { path = "../okena-core" } okena-transport = { path = "../okena-transport", features = ["cancellable-http"] } okena-markdown = { path = "../okena-markdown", optional = true } +okena-highlight = { path = "../okena-highlight", optional = true } okena-ui = { path = "../okena-ui", optional = true } gpui = { git = "https://github.com/zed-industries/zed", package = "gpui", optional = true } gpui-component = { git = "https://github.com/longbridge/gpui-component", package = "gpui-component", optional = true } +# The highlighting itself lives in okena-highlight; syntect stays here for the +# `SyntaxSet` the viewers hold and pass back in. syntect = { version = "5", features = ["default-fancy"], optional = true } -two-face = { version = "0.5", default-features = false, features = ["syntect-fancy"], optional = true } - -# Markdown highlighting: tree-sitter is ~20x faster than syntect's Markdown -# grammar on large files. tree-sitter (0.26) and streaming-iterator are already -# in the workspace via gpui-component, so this only adds the grammar crate. -tree-sitter = { version = "0.26", optional = true } -tree-sitter-md = { version = "0.5", features = ["parser"], optional = true } -streaming-iterator = { version = "0.1", optional = true } grep-searcher = "0.1" grep-regex = "0.1" diff --git a/crates/okena-files/src/code_view.rs b/crates/okena-files/src/code_view.rs index 5cfa6df24..5b3198e93 100644 --- a/crates/okena-files/src/code_view.rs +++ b/crates/okena-files/src/code_view.rs @@ -6,7 +6,7 @@ //! - Text selection utilities use crate::selection::SelectionState; -use crate::syntax::{HighlightedLine, HighlightedSpan}; +use crate::syntax::HighlightedLine; use gpui::*; /// Type alias for code selection (line index, column). @@ -93,89 +93,10 @@ pub fn update_scrollbar_drag( .set_offset(point(px(0.0), px(-new_scroll))); } -/// Build a StyledText with optional background highlights (e.g. selection or word-level diff). -/// Splits syntax color highlights at background range boundaries to produce -/// non-overlapping highlights (required by `StyledText::compute_runs`). -pub fn build_styled_text_with_backgrounds( - spans: &[HighlightedSpan], - bg_ranges: &[(std::ops::Range, Hsla)], -) -> StyledText { - let mut text = String::new(); - let mut highlights = Vec::new(); - - for span in spans { - text.push_str(&span.text); - } - - if bg_ranges.is_empty() { - // Fast path: no background highlights, just syntax colors - let mut offset = 0; - for span in spans { - let start = offset; - offset += span.text.len(); - if start < offset { - highlights.push(( - start..offset, - HighlightStyle { - color: Some(span.color.into()), - ..Default::default() - }, - )); - } - } - } else { - // Split syntax spans at background range boundaries so no highlights overlap - let mut offset = 0; - for span in spans { - let span_start = offset; - let span_end = offset + span.text.len(); - offset = span_end; - - if span_start >= span_end { - continue; - } - - // Collect boundary points from bg_ranges that fall within this span - let mut boundaries = vec![span_start]; - for (br, _) in bg_ranges { - if br.start > span_start && br.start < span_end { - boundaries.push(br.start); - } - if br.end > span_start && br.end < span_end { - boundaries.push(br.end); - } - } - boundaries.push(span_end); - boundaries.sort(); - boundaries.dedup(); - - for window in boundaries.windows(2) { - let sub_start = window[0]; - let sub_end = window[1]; - if sub_start >= sub_end { - continue; - } - - let mut style = HighlightStyle { - color: Some(span.color.into()), - ..Default::default() - }; - - // Apply background if this sub-range falls within any background range - for (br, bg_color) in bg_ranges { - if sub_start >= br.start && sub_end <= br.end { - style.background_color = Some(*bg_color); - break; - } - } - - highlights.push((sub_start..sub_end, style)); - } - } - } - - StyledText::new(text).with_highlights(highlights) -} +// Lives in `okena-highlight` (shared with the markdown renderer); re-exported so +// the code viewers keep their `code_view::build_styled_text_with_backgrounds` +// import. +pub use okena_highlight::styled::build_styled_text_with_backgrounds; /// Compute selection background ranges for a single line. /// diff --git a/crates/okena-files/src/lib.rs b/crates/okena-files/src/lib.rs index 7b784064f..c22e97405 100644 --- a/crates/okena-files/src/lib.rs +++ b/crates/okena-files/src/lib.rs @@ -24,11 +24,11 @@ pub mod in_page_search; #[cfg(feature = "gpui")] pub mod list_overlay; #[cfg(feature = "gpui")] -pub mod markdown_highlight; -#[cfg(feature = "gpui")] pub mod selection; +// Highlighting lives in `okena-highlight` (shared with the markdown renderer); +// re-exported here so the viewers keep their `okena_files::syntax` paths. #[cfg(feature = "gpui")] -pub mod syntax; +pub use okena_highlight::{markdown_highlight, syntax}; // `theme` re-exports gpui theme helpers from `okena-ui` (a gpui crate) and is // consumed only by gpui code, so it is gated with the rest of the viewer even // though it holds no rendering itself. diff --git a/crates/okena-highlight/Cargo.toml b/crates/okena-highlight/Cargo.toml new file mode 100644 index 000000000..5bc790533 --- /dev/null +++ b/crates/okena-highlight/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "okena-highlight" +version = "0.1.0" +edition = "2024" +license = "MIT" + +[dependencies] +gpui = { git = "https://github.com/zed-industries/zed", package = "gpui" } + +syntect = { version = "5", features = ["default-fancy"] } +two-face = { version = "0.5", default-features = false, features = ["syntect-fancy"] } + +# Markdown highlighting: tree-sitter is ~20x faster than syntect's Markdown +# grammar on large files. tree-sitter (0.26) and streaming-iterator are already +# in the workspace via gpui-component, so this only adds the grammar crate. +tree-sitter = "0.26" +tree-sitter-md = { version = "0.5", features = ["parser"] } +streaming-iterator = "0.1" diff --git a/crates/okena-highlight/src/lib.rs b/crates/okena-highlight/src/lib.rs new file mode 100644 index 000000000..949bafbd1 --- /dev/null +++ b/crates/okena-highlight/src/lib.rs @@ -0,0 +1,11 @@ +#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))] + +//! Syntax highlighting shared by every viewer that shows code: the file viewer, +//! the diff viewer, and markdown code blocks. +//! +//! It sits below `okena-files` and `okena-markdown` so all three can share one +//! `SyntaxSet` — loading it is expensive and a second copy would be megabytes. + +pub mod markdown_highlight; +pub mod styled; +pub mod syntax; diff --git a/crates/okena-files/src/markdown_highlight.rs b/crates/okena-highlight/src/markdown_highlight.rs similarity index 97% rename from crates/okena-files/src/markdown_highlight.rs rename to crates/okena-highlight/src/markdown_highlight.rs index db546ec79..fe09c1130 100644 --- a/crates/okena-files/src/markdown_highlight.rs +++ b/crates/okena-highlight/src/markdown_highlight.rs @@ -19,7 +19,7 @@ use crate::syntax::{ HighlightedLine, HighlightedSpan, default_text_color, highlight_line, load_syntax_theme, - map_extension_to_syntax, + syntax_for_language, }; use gpui::Rgba; use std::collections::HashMap; @@ -229,7 +229,7 @@ fn collect_code_blocks( let Some(code) = content.get(start..end.min(content.len())) else { continue; }; - let syntax = syntax_for_lang(lang, syntax_set); + let syntax = syntax_for_language(lang, syntax_set); let mut highlighter = HighlightLines::new(syntax, theme); let first_line = line_of(line_starts, start); for (i, line) in LinesWithEndings::from(code).enumerate() { @@ -239,18 +239,6 @@ fn collect_code_blocks( } } -/// Resolve a fenced-code info-string language to a syntect syntax, falling back -/// to plain text. Mirrors [`crate::syntax::get_syntax_for_path`]'s mapping. -fn syntax_for_lang<'a>( - lang: &str, - syntax_set: &'a SyntaxSet, -) -> &'a syntect::parsing::SyntaxReference { - map_extension_to_syntax(lang) - .and_then(|mapped| syntax_set.find_syntax_by_extension(mapped)) - .or_else(|| syntax_set.find_syntax_by_token(lang)) - .unwrap_or_else(|| syntax_set.find_syntax_plain_text()) -} - /// Build spans for one (non-code) line from the per-byte colour buffer, /// coalescing equal-coloured runs, expanding tabs, and dropping line endings. fn line_spans( diff --git a/crates/okena-highlight/src/styled.rs b/crates/okena-highlight/src/styled.rs new file mode 100644 index 000000000..d05b06313 --- /dev/null +++ b/crates/okena-highlight/src/styled.rs @@ -0,0 +1,91 @@ +//! Turning highlighted spans into a GPUI `StyledText`. +//! +//! Shared by every code surface: the file viewer, the diff viewer, and markdown +//! code blocks all colour the same span data the same way. + +use crate::syntax::HighlightedSpan; +use gpui::*; + +/// Build a StyledText with optional background highlights (e.g. selection or word-level diff). +/// Splits syntax color highlights at background range boundaries to produce +/// non-overlapping highlights (required by `StyledText::compute_runs`). +pub fn build_styled_text_with_backgrounds( + spans: &[HighlightedSpan], + bg_ranges: &[(std::ops::Range, Hsla)], +) -> StyledText { + let mut text = String::new(); + let mut highlights = Vec::new(); + + for span in spans { + text.push_str(&span.text); + } + + if bg_ranges.is_empty() { + // Fast path: no background highlights, just syntax colors + let mut offset = 0; + for span in spans { + let start = offset; + offset += span.text.len(); + if start < offset { + highlights.push(( + start..offset, + HighlightStyle { + color: Some(span.color.into()), + ..Default::default() + }, + )); + } + } + } else { + // Split syntax spans at background range boundaries so no highlights overlap + let mut offset = 0; + for span in spans { + let span_start = offset; + let span_end = offset + span.text.len(); + offset = span_end; + + if span_start >= span_end { + continue; + } + + // Collect boundary points from bg_ranges that fall within this span + let mut boundaries = vec![span_start]; + for (br, _) in bg_ranges { + if br.start > span_start && br.start < span_end { + boundaries.push(br.start); + } + if br.end > span_start && br.end < span_end { + boundaries.push(br.end); + } + } + boundaries.push(span_end); + boundaries.sort(); + boundaries.dedup(); + + for window in boundaries.windows(2) { + let sub_start = window[0]; + let sub_end = window[1]; + if sub_start >= sub_end { + continue; + } + + let mut style = HighlightStyle { + color: Some(span.color.into()), + ..Default::default() + }; + + // Apply background if this sub-range falls within any background range + for (br, bg_color) in bg_ranges { + if sub_start >= br.start && sub_end <= br.end { + style.background_color = Some(*bg_color); + break; + } + } + + highlights.push((sub_start..sub_end, style)); + } + } + } + + StyledText::new(text).with_highlights(highlights) +} diff --git a/crates/okena-files/src/syntax.rs b/crates/okena-highlight/src/syntax.rs similarity index 78% rename from crates/okena-files/src/syntax.rs rename to crates/okena-highlight/src/syntax.rs index ed6ae8215..8fcbbae2c 100644 --- a/crates/okena-files/src/syntax.rs +++ b/crates/okena-highlight/src/syntax.rs @@ -1,14 +1,14 @@ //! Shared syntax highlighting utilities. //! //! Provides types and functions for syntax highlighting that can be used -//! across different viewers (file viewer, diff viewer, etc.). +//! across different viewers (file viewer, diff viewer, markdown code blocks). use gpui::Rgba; use std::path::Path; use std::sync::{Arc, OnceLock}; use syntect::easy::HighlightLines; use syntect::highlighting::Theme; -use syntect::parsing::SyntaxSet; +use syntect::parsing::{SyntaxReference, SyntaxSet}; use syntect::util::LinesWithEndings; /// Global cached syntax set with extended syntaxes (including TypeScript/TSX). @@ -135,10 +135,7 @@ pub fn map_extension_to_syntax(ext: &str) -> Option<&'static str> { } /// Get syntax reference for a file path. -pub fn get_syntax_for_path<'a>( - path: &Path, - syntax_set: &'a SyntaxSet, -) -> &'a syntect::parsing::SyntaxReference { +pub fn get_syntax_for_path<'a>(path: &Path, syntax_set: &'a SyntaxSet) -> &'a SyntaxReference { let ext = path.extension().and_then(|e| e.to_str()); ext.and_then(map_extension_to_syntax) @@ -213,6 +210,48 @@ pub fn highlight_line( } } +/// Resolve a fenced-code info string (`rust`, `ts`, `sh`) to a syntect syntax. +/// `None` when the language is unknown, so callers can tell "highlight as plain +/// text" apart from "leave this block alone". Mirrors [`get_syntax_for_path`]'s +/// mapping. +pub fn find_syntax_for_language<'a>( + lang: &str, + syntax_set: &'a SyntaxSet, +) -> Option<&'a SyntaxReference> { + map_extension_to_syntax(lang) + .and_then(|mapped| syntax_set.find_syntax_by_extension(mapped)) + .or_else(|| syntax_set.find_syntax_by_token(lang)) +} + +/// Like [`find_syntax_for_language`], falling back to plain text. +pub fn syntax_for_language<'a>(lang: &str, syntax_set: &'a SyntaxSet) -> &'a SyntaxReference { + find_syntax_for_language(lang, syntax_set) + .unwrap_or_else(|| syntax_set.find_syntax_plain_text()) +} + +/// Highlight a fenced markdown code block, one entry per line of `code`. +/// +/// A fence with no language, or one syntect does not know, comes back empty: +/// plain-text highlighting would repaint the block in the syntax theme's +/// foreground, and a code block with nothing to colour should keep the +/// document's own text colour instead. +/// +/// Unlike [`highlight_content`] this loads the shared `SyntaxSet` itself — the +/// markdown renderer has no file open and so holds no set of its own. +pub fn highlight_code_block( + code: &str, + language: Option<&str>, + is_dark: bool, +) -> Vec { + let syntax_set = load_syntax_set(); + let Some(syntax) = language.and_then(|lang| find_syntax_for_language(lang, &syntax_set)) else { + return Vec::new(); + }; + // Tabs stay as-is: the markdown renderer maps selection offsets onto the + // raw code text, so a span must keep the character count of its line. + highlight_lines(code, syntax, &syntax_set, 0, is_dark, false) +} + /// Highlight file content and return a vector of highlighted lines. /// /// # Arguments @@ -236,7 +275,35 @@ pub fn highlight_content( ); } - let syntax = get_syntax_for_path(path, syntax_set); + highlight_lines( + content, + get_syntax_for_path(path, syntax_set), + syntax_set, + max_lines, + is_dark, + true, + ) +} + +/// Highlight `content` line by line, merging adjacent spans of the same colour. +/// `expand_tabs` turns each tab into four spaces — right for a viewer that owns +/// its own layout, wrong where the caller indexes back into the original text. +fn highlight_lines( + content: &str, + syntax: &SyntaxReference, + syntax_set: &SyntaxSet, + max_lines: usize, + is_dark: bool, + expand_tabs: bool, +) -> Vec { + let expand = |text: &str| { + if expand_tabs { + text.replace('\t', " ") + } else { + text.to_string() + } + }; + let theme = load_syntax_theme(is_dark); let mut highlighter = HighlightLines::new(syntax, theme); let default_color = default_text_color(is_dark); @@ -263,9 +330,7 @@ pub fn highlight_content( }; // Pre-process text: remove newlines, expand tabs - let processed = text - .trim_end_matches(&['\n', '\r'][..]) - .replace('\t', " "); + let processed = expand(text.trim_end_matches(&['\n', '\r'][..])); if processed.is_empty() { continue; @@ -292,9 +357,7 @@ pub fn highlight_content( (merged, plain) } Err(_) => { - let text = line - .trim_end_matches(&['\n', '\r'][..]) - .replace('\t', " "); + let text = expand(line.trim_end_matches(&['\n', '\r'][..])); ( vec![HighlightedSpan { color: default_color, From 3085b6f6bc123d3ebcbbb905cee0faa01637b44d Mon Sep 17 00:00:00 2001 From: David Matejka Date: Tue, 18 Aug 2026 16:09:05 +0200 Subject: [PATCH 2/2] feat(markdown): colour fenced code blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rendered markdown file showed every code block in one flat colour, while the same file's source view next to it was fully highlighted. There was no reason for the difference beyond where the highlighter lived. `Node::CodeBlock` now carries the spans for each of its lines, filled by `MarkdownDocument::highlight_code_blocks`. That is a separate step from parsing rather than part of it: the colours come from the syntax theme, and the parser has no business knowing whether the app is dark or light. The file viewer calls it after parsing and again from `update_config` when the theme flips — the same place the source view re-highlights. Selection is what constrains the shape of the spans. Offsets are character counts into the raw code, so each line's spans must reproduce that line verbatim; a test pins it. The line is drawn as one `StyledText` with the selection as a background run, instead of the three plain divs a selected line used to split into. Lines with no spans keep the old path, so an unlabelled fence still renders in the document's text colour. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AWSp87PWoqnyB6prN3U3qD --- Cargo.lock | 1 + crates/okena-files/src/file_viewer/loading.rs | 4 +- crates/okena-files/src/file_viewer/mod.rs | 5 ++ crates/okena-markdown/Cargo.toml | 1 + crates/okena-markdown/src/lib.rs | 80 +++++++++++++++++++ crates/okena-markdown/src/parser.rs | 1 + crates/okena-markdown/src/render.rs | 65 +++++++++++++-- crates/okena-markdown/src/types.rs | 8 ++ 8 files changed, 158 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7257a89cd..fce060e44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6331,6 +6331,7 @@ dependencies = [ "gpui", "gpui-component", "okena-core", + "okena-highlight", "okena-ui", "pulldown-cmark", "serde_yaml_ng", diff --git a/crates/okena-files/src/file_viewer/loading.rs b/crates/okena-files/src/file_viewer/loading.rs index 7789cec86..b0dd39c54 100644 --- a/crates/okena-files/src/file_viewer/loading.rs +++ b/crates/okena-files/src/file_viewer/loading.rs @@ -256,7 +256,9 @@ impl FileViewerTab { self.content = content; self.do_highlight_content(&self.file_path.clone(), syntax_set, is_dark); if self.is_markdown { - self.markdown_doc = Some(MarkdownDocument::parse(&self.content)); + let mut doc = MarkdownDocument::parse(&self.content); + doc.highlight_code_blocks(is_dark); + self.markdown_doc = Some(doc); } } Ok(LoadedContent::Image { decoded, source }) => { diff --git a/crates/okena-files/src/file_viewer/mod.rs b/crates/okena-files/src/file_viewer/mod.rs index abc415ed6..08a9a1213 100644 --- a/crates/okena-files/src/file_viewer/mod.rs +++ b/crates/okena-files/src/file_viewer/mod.rs @@ -980,6 +980,11 @@ impl FileViewer { // source-view XML), so they need the rehighlight too. if rehighlight && !tab.is_font && (!tab.is_image || tab.is_svg) { tab.do_highlight_content(&tab.file_path.clone(), &self.syntax_set, self.is_dark); + // The rendered markdown view carries its own highlighted code + // blocks, separate from the source view's lines. + if let Some(doc) = tab.markdown_doc.as_mut() { + doc.highlight_code_blocks(self.is_dark); + } } } } diff --git a/crates/okena-markdown/Cargo.toml b/crates/okena-markdown/Cargo.toml index 70be2aa03..bfa001f12 100644 --- a/crates/okena-markdown/Cargo.toml +++ b/crates/okena-markdown/Cargo.toml @@ -6,6 +6,7 @@ license = "MIT" [dependencies] okena-core = { path = "../okena-core" } +okena-highlight = { path = "../okena-highlight" } okena-ui = { path = "../okena-ui" } gpui = { git = "https://github.com/zed-industries/zed", package = "gpui" } diff --git a/crates/okena-markdown/src/lib.rs b/crates/okena-markdown/src/lib.rs index 7ab7ead25..e75f16bb5 100644 --- a/crates/okena-markdown/src/lib.rs +++ b/crates/okena-markdown/src/lib.rs @@ -51,3 +51,83 @@ pub struct MarkdownDocument { /// Flat text representation of all visible content pub plain_text: String, } + +impl MarkdownDocument { + /// Syntax-highlight every fenced code block for the given theme. + /// + /// Kept out of `parse` because the colours come from the syntax theme, which + /// the parser has no business knowing: the viewer calls this after parsing + /// and again whenever the theme flips between dark and light. A document + /// that never gets the call renders its code blocks in the document text + /// colour, exactly as before. + pub fn highlight_code_blocks(&mut self, is_dark: bool) { + for node in &mut self.nodes { + if let Node::CodeBlock { + language, + code, + highlighted, + } = node + { + *highlighted = okena_highlight::syntax::highlight_code_block( + code, + language.as_deref(), + is_dark, + ) + .into_iter() + .map(|line| line.spans) + .collect(); + } + } + } +} + +#[cfg(test)] +mod tests { + // Named imports, not a glob: `use super::*` would pull in gpui's own `test` + // macro and shadow the one these tests need. + use super::{MarkdownDocument, Node}; + + /// The spans of a line must hold exactly that line's characters — tabs and + /// all. Rendering maps a character-offset selection onto them, so a span set + /// that drops or rewrites characters silently shifts the selection. + #[test] + fn spans_reproduce_each_line_verbatim() { + let mut doc = MarkdownDocument::parse("```rust\nfn main() {\n\tlet x = 1;\n}\n```\n"); + doc.highlight_code_blocks(true); + + let Some(Node::CodeBlock { + code, highlighted, .. + }) = doc.nodes.first() + else { + panic!("expected a code block"); + }; + + let lines: Vec<&str> = code.lines().collect(); + assert_eq!(highlighted.len(), lines.len()); + for (spans, line) in highlighted.iter().zip(&lines) { + let joined: String = spans.iter().map(|s| s.text.as_str()).collect(); + assert_eq!(&joined, line); + } + // More than one colour, or nothing was actually highlighted. + let colors: Vec<_> = highlighted[0].iter().map(|s| s.color.r).collect(); + assert!(colors.len() > 1, "expected `fn main() {{` to be coloured"); + } + + /// A fence with no language, or one syntect cannot place, is left alone so + /// it keeps the document's text colour instead of the syntax theme's. + #[test] + fn unknown_and_missing_languages_stay_unhighlighted() { + for md in [ + "```\nplain text\n```\n", + "```notalanguage\nplain text\n```\n", + ] { + let mut doc = MarkdownDocument::parse(md); + doc.highlight_code_blocks(true); + + let Some(Node::CodeBlock { highlighted, .. }) = doc.nodes.first() else { + panic!("expected a code block"); + }; + assert!(highlighted.is_empty(), "{md:?} should not be highlighted"); + } + } +} diff --git a/crates/okena-markdown/src/parser.rs b/crates/okena-markdown/src/parser.rs index 9ec553d58..c2e27b540 100644 --- a/crates/okena-markdown/src/parser.rs +++ b/crates/okena-markdown/src/parser.rs @@ -122,6 +122,7 @@ impl MarkdownDocument { nodes.push(Node::CodeBlock { language: code_block_lang.take(), code: std::mem::take(&mut code_block_content), + highlighted: Vec::new(), }); in_code_block = false; } diff --git a/crates/okena-markdown/src/render.rs b/crates/okena-markdown/src/render.rs index 3c2336578..6c5f7666b 100644 --- a/crates/okena-markdown/src/render.rs +++ b/crates/okena-markdown/src/render.rs @@ -4,6 +4,8 @@ use gpui::prelude::FluentBuilder; use gpui::*; use gpui_component::{h_flex, v_flex}; use okena_core::theme::ThemeColors; +use okena_highlight::styled::build_styled_text_with_backgrounds; +use okena_highlight::syntax::HighlightedSpan; use okena_ui::code_block::code_block_container; use okena_ui::tokens::ui_text_md; @@ -18,6 +20,33 @@ use super::{MarkdownDocument, RenderedNode}; /// its own selectable element), so this stands in for `line_height`. const CODE_LINE_HEIGHT: Pixels = px(20.0); +/// One syntax-highlighted code line, drawn as a single text run so indentation +/// and wide glyphs measure the same as the source. +/// +/// `selection` is a character range within the line; the spans hold the line's +/// characters unchanged (tabs included), so the range converts straight to the +/// byte offsets `build_styled_text_with_backgrounds` expects. +fn highlighted_code_line( + line: &str, + spans: &[HighlightedSpan], + selection: Option<(usize, usize)>, + selection_bg: Rgba, +) -> StyledText { + let byte_at = |char_idx: usize| { + line.char_indices() + .nth(char_idx) + .map(|(byte, _)| byte) + .unwrap_or(line.len()) + }; + let bg_ranges = match selection { + Some((start, end)) if start < end => { + vec![(byte_at(start)..byte_at(end), selection_bg.into())] + } + _ => Vec::new(), + }; + build_styled_text_with_backgrounds(spans, &bg_ranges) +} + impl MarkdownDocument { /// Number of top-level blocks in the document. Each maps to one list item. pub fn node_count(&self) -> usize { @@ -55,13 +84,17 @@ impl MarkdownDocument { }); let rendered = match node { - Node::CodeBlock { language, code } => { + Node::CodeBlock { + language, + code, + highlighted, + } => { // Return code blocks with individual lines for per-line selection let selection_bg = rgba(0x3390ff40); let mut lines = Vec::new(); let mut line_offset = offset; - for line in code.lines() { + for (line_idx, line) in code.lines().enumerate() { let line_len = char_len(line); let line_end = line_offset + line_len + 1; // +1 for newline @@ -75,7 +108,15 @@ impl MarkdownDocument { } }); - let line_div = if let Some((sel_start, sel_end)) = line_sel { + let spans = highlighted.get(line_idx).map(Vec::as_slice).unwrap_or(&[]); + let line_div = if !spans.is_empty() { + div().h(CODE_LINE_HEIGHT).child(highlighted_code_line( + line, + spans, + line_sel, + selection_bg, + )) + } else if let Some((sel_start, sel_end)) = line_sel { let (before, selected, after) = slice_by_chars(line, sel_start, sel_end); div() .h(CODE_LINE_HEIGHT) @@ -341,14 +382,18 @@ impl MarkdownDocument { Node::Paragraph { children } => { Self::render_inlines_with_selection(children, t, cx, selection).w_full() } - Node::CodeBlock { language, code } => { + Node::CodeBlock { + language, + code, + highlighted, + } => { let selection_bg = rgba(0x3390ff40); // Render code lines with selection let mut code_lines: Vec
= Vec::new(); let mut offset = 0usize; - for line in code.lines() { + for (line_idx, line) in code.lines().enumerate() { let line_len = char_len(line); let line_end = offset + line_len + 1; // +1 for newline @@ -360,7 +405,15 @@ impl MarkdownDocument { } }); - let line_div = if let Some((sel_start, sel_end)) = line_sel { + let spans = highlighted.get(line_idx).map(Vec::as_slice).unwrap_or(&[]); + let line_div = if !spans.is_empty() { + div().h(CODE_LINE_HEIGHT).child(highlighted_code_line( + line, + spans, + line_sel, + selection_bg, + )) + } else if let Some((sel_start, sel_end)) = line_sel { let (before, selected, after) = slice_by_chars(line, sel_start, sel_end); div() .h(CODE_LINE_HEIGHT) diff --git a/crates/okena-markdown/src/types.rs b/crates/okena-markdown/src/types.rs index 603f769e5..3b5f53cba 100644 --- a/crates/okena-markdown/src/types.rs +++ b/crates/okena-markdown/src/types.rs @@ -1,5 +1,7 @@ //! AST types and utility functions for the markdown renderer. +use okena_highlight::syntax::HighlightedSpan; + /// A node in the markdown AST. #[derive(Clone)] pub(crate) enum Node { @@ -13,6 +15,12 @@ pub(crate) enum Node { CodeBlock { language: Option, code: String, + /// Syntax-highlighted spans, one entry per line of `code`. Empty until + /// `MarkdownDocument::highlight_code_blocks` runs — it needs the theme, + /// which parsing does not have — and for languages syntect can't place. + /// Each line's spans hold exactly the characters of that line, so + /// selection offsets map onto them unchanged. + highlighted: Vec>, }, List { ordered: bool,