From 977da7fcb1611e7ee69879836193ae4b0c4c8fbb Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 16:06:32 +0200 Subject: [PATCH 01/11] feat(cli): add cnd output format to typst compile Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/typst-cli/Cargo.toml | 1 + crates/typst-cli/src/args.rs | 6 +- crates/typst-cli/src/compile.rs | 41 +++++++++++++- crates/typst-cli/src/watch.rs | 3 +- crates/typst-cli/src/world.rs | 40 +++++++++----- crates/typst-cli/tests/cnd.rs | 97 +++++++++++++++++++++++++++++++++ crates/typst-cnd/src/world.rs | 10 +++- 8 files changed, 177 insertions(+), 22 deletions(-) create mode 100644 crates/typst-cli/tests/cnd.rs diff --git a/Cargo.lock b/Cargo.lock index f32acc3b81e1..b97a1e3b4634 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3330,6 +3330,7 @@ dependencies = [ "toml", "typst", "typst-bundle", + "typst-cnd", "typst-dev-assets", "typst-eval", "typst-html", diff --git a/crates/typst-cli/Cargo.toml b/crates/typst-cli/Cargo.toml index 7d9bab8d1763..f7186c49a992 100644 --- a/crates/typst-cli/Cargo.toml +++ b/crates/typst-cli/Cargo.toml @@ -20,6 +20,7 @@ doc = false [dependencies] typst = { workspace = true } typst-bundle = { workspace = true } +typst-cnd = { workspace = true } typst-eval = { workspace = true } typst-layout = { workspace = true } typst-html = { workspace = true } diff --git a/crates/typst-cli/src/args.rs b/crates/typst-cli/src/args.rs index 2429c4f46d99..498e56136579 100644 --- a/crates/typst-cli/src/args.rs +++ b/crates/typst-cli/src/args.rs @@ -340,8 +340,8 @@ pub struct CompileArgs { #[clap(value_parser = input_value_parser(), value_hint = ValueHint::FilePath)] pub input: Input, - /// Path to output file (PDF, PNG, SVG, or HTML). Use `-` to write output to - /// stdout. + /// Path to output file (PDF, PNG, SVG, HTML, or CND). Use `-` to write + /// output to stdout. /// /// For output formats emitting one file per page (PNG & SVG), a page number /// template must be present if the source document renders to multiple @@ -664,6 +664,8 @@ pub enum OutputFormat { Svg, Html, Bundle, + /// A CND: the document's semantic structure as JSON. + Cnd, } impl OutputFormat { diff --git a/crates/typst-cli/src/compile.rs b/crates/typst-cli/src/compile.rs index 64760061e341..ce11fa455994 100644 --- a/crates/typst-cli/src/compile.rs +++ b/crates/typst-cli/src/compile.rs @@ -39,10 +39,11 @@ use typst_kit::server::HttpServer; pub fn compile(command: &'static CompileCommand) -> HintedStrResult<()> { let mut timer = Timer::new_or_placeholder(command.args.timings.clone()); let mut config = CompileConfig::new(command)?; - let mut world = SystemWorld::new( + let mut world = SystemWorld::new_with_format( Some(&command.args.input), &command.args.world, &command.args.process, + Some(config.output_format), ) .map_err(|err| eco_format!("{err}"))?; timer.record(&mut world, |world| compile_once(world, &mut config))? @@ -58,7 +59,7 @@ pub struct CompileConfig { pub fullscreen: bool, /// Path to input Typst file or stdin. pub input: Input, - /// Path to output file (PDF, PNG, SVG, or HTML). + /// Path to output file (PDF, PNG, SVG, HTML, or CND). pub output: Output, /// The format of the output file. pub output_format: OutputFormat, @@ -119,6 +120,7 @@ impl CompileConfig { Some(ext) if ext.eq_ignore_ascii_case("png") => OutputFormat::Png, Some(ext) if ext.eq_ignore_ascii_case("svg") => OutputFormat::Svg, Some(ext) if ext.eq_ignore_ascii_case("html") => OutputFormat::Html, + Some(ext) if ext.eq_ignore_ascii_case("cnd") => OutputFormat::Cnd, _ => bail!( "could not infer output format for path {}.\n\ consider providing the format manually with `--format/-f`", @@ -140,6 +142,7 @@ impl CompileConfig { OutputFormat::Svg => "svg", OutputFormat::Html => "html", OutputFormat::Bundle => "", + OutputFormat::Cnd => "cnd", }, )) }); @@ -363,6 +366,15 @@ fn compile_and_export( } OutputFormat::Bundle => typst::compile::(world) .and_then(|bundle| export_bundle(bundle, config)), + OutputFormat::Cnd => { + let Warned { output, warnings } = + typst::compile::(&*world); + let result = output.and_then(|document| export_cnd(&document, world, config)); + Warned { + output: result.map(|()| vec![config.output.clone()]), + warnings, + } + } } } @@ -382,6 +394,29 @@ fn export_html(document: &HtmlDocument, config: &CompileConfig) -> SourceResult< .at(Span::detached()) } +/// Export to CND. +fn export_cnd( + document: &typst_cnd::CndDocument, + world: &SystemWorld, + config: &CompileConfig, +) -> SourceResult<()> { + let built_at = match config.creation_timestamp { + Some(timestamp) => timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + None => typst_cnd::world::built_at_now(), + }; + let cnd = typst_cnd::cnd_from_document( + document, + typst_cnd::world::source_info(world), + built_at, + ); + let json = typst_cnd::cnd_to_json(&cnd)?; + config + .output + .write(json.as_bytes()) + .map_err(|err| eco_format!("failed to write CND file ({err})")) + .at(Span::detached()) +} + /// Export to a paged target format. fn export_paged( document: &PagedDocument, @@ -397,7 +432,7 @@ fn export_paged( OutputFormat::Svg => export_image(document, config, ImageExportFormat::Svg) .at(Span::detached()) .into(), - OutputFormat::Html | OutputFormat::Bundle => unreachable!(), + OutputFormat::Html | OutputFormat::Bundle | OutputFormat::Cnd => unreachable!(), } } diff --git a/crates/typst-cli/src/watch.rs b/crates/typst-cli/src/watch.rs index 1c29e459dc71..87b4bb223f4c 100644 --- a/crates/typst-cli/src/watch.rs +++ b/crates/typst-cli/src/watch.rs @@ -30,10 +30,11 @@ pub fn watch(command: &'static WatchCommand) -> HintedStrResult<()> { // Create the world that serves sources, files, and fonts. // Additionally, if any files do not exist, wait until they do. let mut world = loop { - match SystemWorld::new( + match SystemWorld::new_with_format( Some(&command.args.input), &command.args.world, &command.args.process, + Some(config.output_format), ) { Ok(world) => break world, Err( diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 19d795e070e7..6c8de74ec226 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -19,7 +19,7 @@ use typst_kit::files::{FileLoader, FileStore, FsRoot}; use typst_kit::fonts::FontStore; use typst_kit::packages::SystemPackages; -use crate::args::{Feature, Input, ProcessArgs, WorldArgs}; +use crate::args::{Feature, Input, OutputFormat, ProcessArgs, WorldArgs}; /// A world that provides access to the operating system. pub struct SystemWorld { @@ -38,11 +38,21 @@ pub struct SystemWorld { } impl SystemWorld { - /// Creates a new system world. + /// Creates a new system world for any format but CND. pub fn new( input: Option<&Input>, world_args: &'static WorldArgs, process_args: &ProcessArgs, + ) -> Result { + Self::new_with_format(input, world_args, process_args, None) + } + + /// Creates a new system world whose standard library suits `format`. + pub fn new_with_format( + input: Option<&Input>, + world_args: &'static WorldArgs, + process_args: &ProcessArgs, + format: Option, ) -> Result { // Set up the thread pool. if let Some(jobs) = process_args.jobs { @@ -61,19 +71,23 @@ impl SystemWorld { .map(|(k, v)| (k.as_str().into(), v.as_str().into_value())) .collect(); - let features = + let features: Vec = process_args.features.iter().copied().map(Into::into).collect(); - Library::builder([ - typst_html::FORMAT, - typst_pdf::FORMAT, - typst_svg::FORMAT, - typst_render::FORMAT, - typst_bundle::FORMAT, - ]) - .with_inputs(inputs) - .with_features(features) - .build() + if format == Some(OutputFormat::Cnd) { + typst_cnd::world::cnd_library(inputs, features) + } else { + Library::builder([ + typst_html::FORMAT, + typst_pdf::FORMAT, + typst_svg::FORMAT, + typst_render::FORMAT, + typst_bundle::FORMAT, + ]) + .with_inputs(inputs) + .with_features(features.into_iter().collect()) + .build() + } }; let now = match world_args.creation_timestamp { diff --git a/crates/typst-cli/tests/cnd.rs b/crates/typst-cli/tests/cnd.rs new file mode 100644 index 000000000000..272ba87a4b88 --- /dev/null +++ b/crates/typst-cli/tests/cnd.rs @@ -0,0 +1,97 @@ +//! Integration tests for `typst compile --format cnd`. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use typst_cnd::Cnd; + +fn exec() -> Command { + Command::new(env!("CARGO_BIN_EXE_typst")) +} + +fn write(dir: &Path, rel: &str, data: impl AsRef<[u8]>) -> PathBuf { + let path = dir.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, data).unwrap(); + path +} + +#[track_caller] +fn run_ok(cmd: &mut Command) -> std::process::Output { + let out = cmd.output().unwrap(); + assert!( + out.status.success(), + "command failed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + out +} + +fn read_cnd(path: &Path) -> Cnd { + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +#[test] +fn cnd_format_by_flag_and_by_extension() { + let dir = tempfile::tempdir().unwrap(); + let main = write(dir.path(), "main.typ", "= Hello\n\nWorld."); + + run_ok(exec().arg("compile").arg(&main).arg("--format").arg("cnd")); + let by_flag = read_cnd(&dir.path().join("main.cnd")); + assert!(!by_flag.nodes.is_empty()); + + let out = dir.path().join("explicit.cnd"); + run_ok(exec().arg("compile").arg(&main).arg(&out)); + assert!(!read_cnd(&out).nodes.is_empty()); +} + +#[test] +fn cnd_to_stdout_is_only_json() { + let dir = tempfile::tempdir().unwrap(); + let main = write(dir.path(), "main.typ", "= Hello"); + let out = + run_ok(exec().arg("compile").arg(&main).arg("-").arg("--format").arg("cnd")); + let cnd: Cnd = + serde_json::from_slice(&out.stdout).expect("stdout is exactly one CND"); + assert!(!cnd.nodes.is_empty()); +} + +#[test] +fn cnd_semantics_feature_is_on() { + // Text inside a block body is only located (and so emitted) under + // `Feature::CndSemantics`; without it the paragraph silently vanishes. + let dir = tempfile::tempdir().unwrap(); + let main = write(dir.path(), "main.typ", "#block[Inside a block.]"); + run_ok(exec().arg("compile").arg(&main).arg("--format").arg("cnd")); + let json = std::fs::read_to_string(dir.path().join("main.cnd")).unwrap(); + assert!(json.contains("Inside a block."), "{json}"); +} + +#[test] +fn root_flag_resolves_absolute_imports_from_a_nested_main() { + let dir = tempfile::tempdir().unwrap(); + write(dir.path(), "shared.typ", "#let greeting = [Shared text.]"); + let main = write( + dir.path(), + "chapters/main.typ", + "#import \"/shared.typ\": greeting\n#greeting", + ); + let out = dir.path().join("out.cnd"); + run_ok( + exec() + .arg("compile") + .arg(&main) + .arg(&out) + .arg("--root") + .arg(dir.path()), + ); + let cnd = read_cnd(&out); + let json = serde_json::to_string(&cnd).unwrap(); + assert!(json.contains("Shared text.")); + // source.uri is the main file's path relative to --root. + assert_eq!( + cnd.source.as_ref().and_then(|source| source.uri.as_deref()), + Some("chapters/main.typ") + ); +} diff --git a/crates/typst-cnd/src/world.rs b/crates/typst-cnd/src/world.rs index 16b4c0bd91b8..12f25a78ac3e 100644 --- a/crates/typst-cnd/src/world.rs +++ b/crates/typst-cnd/src/world.rs @@ -26,13 +26,17 @@ pub struct CndWorld { now: Time, } -fn build_library(inputs: Dict) -> Library { +/// Builds the standard library a CND compilation needs: `inputs` as +/// `sys.inputs`, `features` plus [`Feature::CndSemantics`], the `pdf` +/// format's bindings, and the deprecated `cnd` global. +pub fn cnd_library(inputs: Dict, features: impl IntoIterator) -> Library { // `Feature::CndSemantics` makes every fully-inline fragment body (a // `block[…]`, a grid cell, a `place`, a `box`) realize into a real, // locatable `ParElem`. Without it that text is laid out but never // located, so `Introspector::query` — the only source the CND emit // pipeline reads — cannot see it and the content is silently dropped. - let features = Features::from_iter([Feature::CndSemantics]); + let features: Features = + features.into_iter().chain([Feature::CndSemantics]).collect(); // Upstream #8496 made every export format opt-in: a format's bindings // (`pdf.embed`, `pdf.header-cell`, …) now exist only if the format is // registered here. Before it, `pdf` was defined unconditionally, so @@ -74,7 +78,7 @@ impl CndWorld { Ok(Self { main, - library: LazyHash::new(build_library(inputs)), + library: LazyHash::new(cnd_library(inputs, [])), fonts: LazyLock::new(Box::new(|| { let mut store = FontStore::new(); store.extend(fonts::embedded()); From c2f35e9dd8aa4eb5adc10768b31e7737e7958be9 Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 16:25:22 +0200 Subject: [PATCH 02/11] fix(cnd): serialize state_metadata in a stable key order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The example-suite parity test (added next) caught complex_multipage.typ producing textually different CND output across separate `typst compile` runs of the same document. Root cause: `NodeBase::state_metadata` was a `std::collections::HashMap`, whose iteration order is randomized per process — confirmed by running the CLI three times on the same input and diffing the key order of one node's state_metadata each time. Switch it (and every construction site: metadata.rs, emit/convert.rs, emit/figure.rs, emit/table.rs) to `serde_json::Map`. Without serde_json's `preserve_order` feature (deliberately not enabled anywhere in this workspace, since Cargo feature unification would change map ordering for every crate in the build, including upstream ones), `Map` is backed by a `BTreeMap`, so keys always serialize in the same alphabetical order regardless of process or machine. Nested dicts already went through `serde_json::Map` in `value_to_json`; this makes the top level consistent with them. Adds a regression unit test in metadata.rs asserting alphabetical key order from a dict inserted in non-alphabetical order. Co-Authored-By: Claude Sonnet 5 --- crates/typst-cnd/src/emit/convert.rs | 9 ++--- crates/typst-cnd/src/emit/figure.rs | 2 +- crates/typst-cnd/src/emit/table.rs | 4 +- crates/typst-cnd/src/metadata.rs | 55 +++++++++++++++++++++++++--- crates/typst-cnd/src/model.rs | 16 +++++--- 5 files changed, 68 insertions(+), 18 deletions(-) diff --git a/crates/typst-cnd/src/emit/convert.rs b/crates/typst-cnd/src/emit/convert.rs index 8519145a04ee..056a63a93402 100644 --- a/crates/typst-cnd/src/emit/convert.rs +++ b/crates/typst-cnd/src/emit/convert.rs @@ -67,7 +67,7 @@ pub struct NodeRecord { /// `refs::resolve_refs` (a label/location destination is a /// cross-reference with a custom body). pub link_markers: Vec<(LinkDest, (i64, i64))>, - pub state_metadata: std::collections::HashMap, + pub state_metadata: serde_json::Map, } /// Conversion context shared across the document walk. @@ -978,8 +978,7 @@ pub fn metadata_at( engine: &mut Engine, introspector: &dyn Introspector, location: Location, -) -> typst_library::diag::SourceResult> -{ +) -> typst_library::diag::SourceResult> { crate::metadata::metadata_at(engine, introspector, location) } @@ -1038,7 +1037,7 @@ pub fn make_record( let state_metadata = match location { Some(loc) => metadata_at(engine, introspector, loc)?, - None => std::collections::HashMap::new(), + None => serde_json::Map::new(), }; Ok(NodeRecord { @@ -1130,7 +1129,7 @@ fn set_label(node: &mut CndNode, label: &str) { fn set_metadata( node: &mut CndNode, - metadata: std::collections::HashMap, + metadata: serde_json::Map, ) { node.base_mut().state_metadata = metadata; } diff --git a/crates/typst-cnd/src/emit/figure.rs b/crates/typst-cnd/src/emit/figure.rs index 507cc48eeb3c..a904a0ebeac9 100644 --- a/crates/typst-cnd/src/emit/figure.rs +++ b/crates/typst-cnd/src/emit/figure.rs @@ -103,7 +103,7 @@ fn minimal_record(content: &Content) -> NodeRecord { ref_markers: Vec::new(), link_targets: Vec::new(), link_markers: Vec::new(), - state_metadata: std::collections::HashMap::new(), + state_metadata: serde_json::Map::new(), } } diff --git a/crates/typst-cnd/src/emit/table.rs b/crates/typst-cnd/src/emit/table.rs index 257069ee8177..c52de95a7096 100644 --- a/crates/typst-cnd/src/emit/table.rs +++ b/crates/typst-cnd/src/emit/table.rs @@ -159,7 +159,7 @@ pub fn from_figure_grid( ref_markers: Vec::new(), link_targets: Vec::new(), link_markers: Vec::new(), - state_metadata: std::collections::HashMap::new(), + state_metadata: serde_json::Map::new(), }; let wrapper_id = Uuid::new_v4(); @@ -221,7 +221,7 @@ pub fn convert( /// actually reads. Only a string value is accepted; anything else (wrong /// type, or simply absent) leaves the hint unset rather than guessing. fn content_kind_from_metadata( - metadata: &std::collections::HashMap, + metadata: &serde_json::Map, ) -> Option { match metadata.get("content_kind") { Some(serde_json::Value::String(s)) => Some(s.clone()), diff --git a/crates/typst-cnd/src/metadata.rs b/crates/typst-cnd/src/metadata.rs index 734a35de55ce..c31d07723d08 100644 --- a/crates/typst-cnd/src/metadata.rs +++ b/crates/typst-cnd/src/metadata.rs @@ -1,7 +1,5 @@ //! Reading Typst `state` values for CND metadata export. -use std::collections::HashMap; - use comemo::Track; use typst_library::diag::SourceResult; use typst_library::engine::Engine; @@ -12,19 +10,25 @@ use typst_library::introspection::{Introspector, state_value_at}; use crate::cnd::metadata_state; /// Read accumulated CND metadata at a document location. +/// +/// Returns a `serde_json::Map` (a `BTreeMap` under the hood, since this +/// workspace never enables `serde_json`'s `preserve_order` feature), not a +/// `HashMap`: a `HashMap`'s iteration order is randomized per process, so +/// the same document would serialize its metadata keys in a different order +/// on every run — see the doc comment on `NodeBase::state_metadata`. pub fn metadata_at( engine: &mut Engine, introspector: &dyn Introspector, location: Location, -) -> SourceResult> { +) -> SourceResult> { let state = metadata_state(); let value = state_value_at(&state, engine, introspector.track(), location)?; Ok(dict_value_to_metadata(&value)) } -fn dict_value_to_metadata(value: &Value) -> HashMap { +fn dict_value_to_metadata(value: &Value) -> serde_json::Map { let Value::Dict(dict) = value else { - return HashMap::new(); + return serde_json::Map::new(); }; dict.iter() @@ -57,3 +61,44 @@ fn value_to_json(value: &Value) -> Option { _ => Some(serde_json::Value::String(value.repr().into())), } } + +#[cfg(test)] +mod tests { + use typst_library::foundations::{Value, dict}; + + use super::dict_value_to_metadata; + + /// Regression test for the parity bug found by + /// `cnd_format_matches_the_standalone_exporter_on_every_example` + /// (`typst-cli/tests/cnd.rs`): two compilations of the same document, + /// in two different processes, serialized `state_metadata` with a + /// different key order, because the field was a `HashMap` (random + /// per-process order) rather than a `serde_json::Map` (alphabetical, + /// deterministic across every run and every machine). + /// + /// The dict below inserts keys in a deliberately non-alphabetical + /// order; with 8 keys, a `HashMap`'s random iteration order would only + /// coincidentally sort itself alphabetically about 1 time in 8! (over + /// 40,000), so this reliably caught the bug before the fix. + #[test] + fn state_metadata_keys_serialize_in_a_stable_alphabetical_order() { + let value = Value::Dict(dict! { + "zone" => "z", + "wolf" => "w", + "mango" => "m", + "kite" => "k", + "juno" => "j", + "delta" => "d", + "banana" => "b", + "apple" => "a", + }); + + let metadata = dict_value_to_metadata(&value); + let keys: Vec<&str> = metadata.keys().map(String::as_str).collect(); + + assert_eq!( + keys, + vec!["apple", "banana", "delta", "juno", "kite", "mango", "wolf", "zone"], + ); + } +} diff --git a/crates/typst-cnd/src/model.rs b/crates/typst-cnd/src/model.rs index c64f8472931f..9290352c81b8 100644 --- a/crates/typst-cnd/src/model.rs +++ b/crates/typst-cnd/src/model.rs @@ -5,8 +5,6 @@ //! emitting a CND and running it through the SDK's `validate()`, not by //! construction. -use std::collections::HashMap; - use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -311,8 +309,16 @@ pub struct NodeBase { /// forward family alongside `refs`/`cites`/`footnotes`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub links: Vec, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub state_metadata: HashMap, + /// A plain `serde_json::Map`, not a `HashMap`: without `serde_json`'s + /// `preserve_order` feature (never enabled in this workspace — it would + /// change map ordering for every crate in the build), `Map` is backed + /// by a `BTreeMap`, so keys always serialize in the same alphabetical + /// order. A `HashMap` here would serialize its keys in a random, + /// per-process order, which is exactly what CND's `built_at`/UUID + /// normalization in `typst-cli`'s parity test cannot paper over: the + /// same document would produce a textually different CND on every run. + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub state_metadata: serde_json::Map, pub location: NodeLocation, } @@ -569,7 +575,7 @@ impl NodeBase { cites: Vec::new(), footnotes: Vec::new(), links: Vec::new(), - state_metadata: HashMap::new(), + state_metadata: serde_json::Map::new(), location, } } From d4fa2dc298a2911fe81303b0bcff1317b9eb59cf Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 16:25:37 +0200 Subject: [PATCH 03/11] test(cli): cnd format matches the standalone exporter on the example suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds CndWorld::new_with_options(input, inputs, system_fonts), with new_with_inputs now a thin wrapper over it (system_fonts: true). This lets a test build the same font set --ignore-system-fonts gives the CLI path, so both entry points are comparable with system fonts off — they otherwise discover fonts in a different order, which is a font-setup difference, not an exporter one. cnd_format_matches_the_standalone_exporter_on_every_example compiles every .typ file in typst-cnd/examples/ through both entry points (typst compile --format cnd, and CndWorld + cnd_from_document/ cnd_to_json directly) and asserts the two CNDs are identical after normalizing built_at and node UUIDs (order of first appearance). This caught the state_metadata ordering bug fixed in the previous commit; all 21 examples now agree byte-for-byte, verified stable across five repeated runs. Co-Authored-By: Claude Sonnet 5 --- crates/typst-cli/tests/cnd.rs | 96 +++++++++++++++++++++++++++++++++++ crates/typst-cnd/src/world.rs | 17 ++++++- 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/crates/typst-cli/tests/cnd.rs b/crates/typst-cli/tests/cnd.rs index 272ba87a4b88..fec20b1a5c1b 100644 --- a/crates/typst-cli/tests/cnd.rs +++ b/crates/typst-cli/tests/cnd.rs @@ -95,3 +95,99 @@ fn root_flag_resolves_absolute_imports_from_a_nested_main() { Some("chapters/main.typ") ); } + +/// Every example of the CND suite compiles to the same CND through +/// `typst compile --format cnd` as through the typst-cnd library entry point +/// the standalone binary uses. System fonts are off on both sides: the two +/// entry points discover them in a different order, which is a font-setup +/// difference, not an exporter one. +#[test] +fn cnd_format_matches_the_standalone_exporter_on_every_example() { + let examples = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../typst-cnd/examples"); + let mut files: Vec = std::fs::read_dir(&examples) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "typ")) + .collect(); + files.sort(); + assert!(files.len() >= 9, "expected the example suite, found {}", files.len()); + + let out_dir = tempfile::tempdir().unwrap(); + for path in files { + let world = typst_cnd::world::CndWorld::new_with_options( + &path, + typst::foundations::Dict::default(), + false, + ) + .unwrap(); + let document = typst::compile::(&world) + .output + .unwrap_or_else(|errors| panic!("{}: {errors:?}", path.display())); + let expected = typst_cnd::cnd_to_json(&typst_cnd::cnd_from_document( + &document, + typst_cnd::world::source_info(&world), + "1970-01-01T00:00:00Z".into(), + )) + .unwrap(); + + let out = out_dir.path().join("out.cnd"); + run_ok( + exec() + .arg("compile") + .arg(&path) + .arg(&out) + .arg("--ignore-system-fonts") + .arg("--creation-timestamp") + .arg("0"), + ); + let actual = std::fs::read_to_string(&out).unwrap(); + assert_eq!( + normalize_uuids(&actual), + normalize_uuids(&expected), + "{} differs", + path.display() + ); + } +} + +/// Node ids are random v4 UUIDs, minted per compilation. Replaces each +/// distinct UUID by its order of first appearance, so two compilations of +/// the same document compare equal exactly when their structure, text and +/// cross-references (which reuse the ids) are equal. +fn normalize_uuids(json: &str) -> String { + fn is_uuid(s: &[u8]) -> bool { + s.len() == 36 + && s.iter().enumerate().all(|(i, &b)| match i { + 8 | 13 | 18 | 23 => b == b'-', + _ => b.is_ascii_hexdigit(), + }) + } + let bytes = json.as_bytes(); + let mut seen: std::collections::HashMap<&str, usize> = + std::collections::HashMap::default(); + let mut out = String::with_capacity(json.len()); + let mut i = 0; + while i < bytes.len() { + if i + 36 <= bytes.len() && is_uuid(&bytes[i..i + 36]) { + let next = seen.len(); + let n = *seen.entry(&json[i..i + 36]).or_insert(next); + out.push_str(&format!("uuid-{n}")); + i += 36; + } else { + let ch = json[i..].chars().next().unwrap(); + out.push(ch); + i += ch.len_utf8(); + } + } + out +} + +#[test] +fn normalize_uuids_keeps_references_aligned() { + let a = r#"{"id":"11111111-1111-4111-8111-111111111111","ref":"11111111-1111-4111-8111-111111111111","b":"22222222-2222-4222-8222-222222222222"}"#; + let b = r#"{"id":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","ref":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","b":"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"}"#; + let c = r#"{"id":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","ref":"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb","b":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"}"#; + assert_eq!(normalize_uuids(a), normalize_uuids(b)); + assert_ne!(normalize_uuids(a), normalize_uuids(c)); +} diff --git a/crates/typst-cnd/src/world.rs b/crates/typst-cnd/src/world.rs index 12f25a78ac3e..eb38ffe29976 100644 --- a/crates/typst-cnd/src/world.rs +++ b/crates/typst-cnd/src/world.rs @@ -62,6 +62,17 @@ impl CndWorld { /// called). Empty by default, so [`new`](Self::new) behaves exactly as /// it always has. pub fn new_with_inputs(input: &Path, inputs: Dict) -> Result { + Self::new_with_options(input, inputs, true) + } + + /// Like [`new_with_inputs`](Self::new_with_inputs), but `system_fonts: + /// false` restricts the font set to the embedded fonts, for reproducible + /// comparisons across machines. + pub fn new_with_options( + input: &Path, + inputs: Dict, + system_fonts: bool, + ) -> Result { let root = input .parent() .unwrap_or(Path::new(".")) @@ -79,10 +90,12 @@ impl CndWorld { Ok(Self { main, library: LazyHash::new(cnd_library(inputs, [])), - fonts: LazyLock::new(Box::new(|| { + fonts: LazyLock::new(Box::new(move || { let mut store = FontStore::new(); store.extend(fonts::embedded()); - store.extend(fonts::system()); + if system_fonts { + store.extend(fonts::system()); + } store })), files: FileStore::new(CndFiles { From daf3d05e34fb00680b1c416401480e5a6e2c1c5b Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 16:37:12 +0200 Subject: [PATCH 04/11] feat(cli): add --inputs-file for sys.inputs Loads a JSON object of string values into sys.inputs, merged with --input pairs (--input wins on a key collision since it is applied last). Ports build_inputs/load_inputs_file/json_kind from typst-cnd's standalone binary, with errors as String instead of SourceResult so they compose with WorldCreationError. Co-Authored-By: Claude Sonnet 5 --- crates/typst-cli/src/args.rs | 9 +++ crates/typst-cli/src/inputs.rs | 116 +++++++++++++++++++++++++++++++++ crates/typst-cli/src/main.rs | 1 + crates/typst-cli/src/world.rs | 13 ++-- crates/typst-cli/tests/cnd.rs | 51 +++++++++++++++ 5 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 crates/typst-cli/src/inputs.rs diff --git a/crates/typst-cli/src/args.rs b/crates/typst-cli/src/args.rs index 498e56136579..4d7ee08c76aa 100644 --- a/crates/typst-cli/src/args.rs +++ b/crates/typst-cli/src/args.rs @@ -480,6 +480,15 @@ pub struct WorldArgs { )] pub inputs: Vec<(String, String)>, + /// Load a JSON object of string values into `sys.inputs`. + /// + /// Every value must be a JSON string — often itself serialized JSON, + /// decoded again on the Typst side. Use this instead of many `--input` + /// flags when the data would not fit comfortably on the command line. A + /// key also given with `--input` takes the `--input` value. + #[clap(long = "inputs-file", value_name = "PATH")] + pub inputs_file: Option, + /// Common font arguments. #[clap(flatten)] pub font: FontArgs, diff --git a/crates/typst-cli/src/inputs.rs b/crates/typst-cli/src/inputs.rs new file mode 100644 index 000000000000..267faffe8a33 --- /dev/null +++ b/crates/typst-cli/src/inputs.rs @@ -0,0 +1,116 @@ +//! `sys.inputs` from `--input` pairs and an optional `--inputs-file`. + +use std::path::Path; + +use typst::foundations::{Dict, IntoValue}; + +use crate::args::WorldArgs; + +/// Merges `--inputs-file` and `--input` into one dictionary. On a key +/// collision `--input` wins, because it is applied last. +pub fn sys_inputs(args: &WorldArgs) -> Result { + let mut pairs = Vec::new(); + if let Some(path) = &args.inputs_file { + pairs.extend(load_inputs_file(path)?); + } + pairs.extend(args.inputs.iter().cloned()); + Ok(pairs + .into_iter() + .map(|(k, v)| (k.as_str().into(), v.as_str().into_value())) + .collect()) +} + +/// Loads a JSON object whose values are all strings. Values are taken +/// verbatim (unlike `--input`, which trims): a value is often a serialized +/// document in which whitespace can matter. +pub fn load_inputs_file(path: &Path) -> Result, String> { + let shown = path.display(); + let text = std::fs::read_to_string(path) + .map_err(|err| format!("failed to read inputs file {shown}: {err}"))?; + let value: serde_json::Value = serde_json::from_str(&text) + .map_err(|err| format!("failed to parse inputs file {shown} as JSON: {err}"))?; + let object = value.as_object().ok_or_else(|| { + format!( + "inputs file {shown} must contain a JSON object at its top level, found {}", + json_kind(&value) + ) + })?; + let mut pairs = Vec::with_capacity(object.len()); + for (key, val) in object { + if key.trim().is_empty() { + return Err(format!("inputs file {shown}: keys must not be empty")); + } + let Some(s) = val.as_str() else { + return Err(format!( + "inputs file {shown}: value for key {key:?} must be a string, found {}", + json_kind(val) + )); + }; + pairs.push((key.clone(), s.to_string())); + } + Ok(pairs) +} + +fn json_kind(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "a boolean", + serde_json::Value::Number(_) => "a number", + serde_json::Value::String(_) => "a string", + serde_json::Value::Array(_) => "an array", + serde_json::Value::Object(_) => "an object", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(json: &str) -> tempfile::NamedTempFile { + let f = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(f.path(), json).unwrap(); + f + } + + #[test] + fn reads_string_values() { + let f = file(r#"{"a": "1", "b": "{\"x\": 2}"}"#); + let pairs = load_inputs_file(f.path()).unwrap(); + assert_eq!( + pairs, + vec![("a".into(), "1".into()), ("b".into(), "{\"x\": 2}".into())] + ); + } + + #[test] + fn keeps_whitespace_in_values() { + let f = file(r#"{"a": " padded "}"#); + assert_eq!(load_inputs_file(f.path()).unwrap()[0].1, " padded "); + } + + #[test] + fn rejects_non_objects_non_strings_and_empty_keys() { + assert!( + load_inputs_file(file("[]").path()) + .unwrap_err() + .contains("JSON object") + ); + assert!( + load_inputs_file(file(r#"{"a": 1}"#).path()) + .unwrap_err() + .contains("must be a string") + ); + assert!( + load_inputs_file(file(r#"{" ": "x"}"#).path()) + .unwrap_err() + .contains("must not be empty") + ); + assert!(load_inputs_file(file("{").path()).unwrap_err().contains("as JSON")); + } + + #[test] + fn missing_file_is_an_error() { + let err = load_inputs_file(Path::new("/definitely/not/here.json")).unwrap_err(); + assert!(err.contains("failed to read inputs file"), "{err}"); + } +} diff --git a/crates/typst-cli/src/main.rs b/crates/typst-cli/src/main.rs index 6e47b1c824e3..4ab7e959f47d 100644 --- a/crates/typst-cli/src/main.rs +++ b/crates/typst-cli/src/main.rs @@ -8,6 +8,7 @@ mod fonts; mod greet; mod info; mod init; +mod inputs; mod packages; mod query; mod terminal; diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 6c8de74ec226..2d0ff45eb9de 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -6,7 +6,7 @@ use std::sync::LazyLock; use ecow::{EcoString, eco_format}; use typst::diag::{FileError, FileResult}; -use typst::foundations::{Bytes, Datetime, Dict, Duration, IntoValue, Repr}; +use typst::foundations::{Bytes, Datetime, Duration, Repr}; use typst::syntax::{ FileId, PathError, RootedPath, Source, VirtualPath, VirtualRoot, VirtualizeError, }; @@ -64,12 +64,8 @@ impl SystemWorld { } let library = { - // Convert the input pairs to a dictionary. - let inputs: Dict = world_args - .inputs - .iter() - .map(|(k, v)| (k.as_str().into(), v.as_str().into_value())) - .collect(); + let inputs = crate::inputs::sys_inputs(world_args) + .map_err(WorldCreationError::Inputs)?; let features: Vec = process_args.features.iter().copied().map(Into::into).collect(); @@ -315,6 +311,8 @@ pub enum WorldCreationError { RootNotFound(PathBuf), /// The requested creation timestamp was invalid. InvalidTimestamp, + /// The `--inputs-file` could not be used. + Inputs(String), /// Another type of I/O error. Io(io::Error), } @@ -343,6 +341,7 @@ impl fmt::Display for WorldCreationError { WorldCreationError::InvalidTimestamp => { write!(f, "creation timestamp out of range") } + WorldCreationError::Inputs(message) => write!(f, "{message}"), WorldCreationError::Io(err) => write!(f, "{err}"), } } diff --git a/crates/typst-cli/tests/cnd.rs b/crates/typst-cli/tests/cnd.rs index fec20b1a5c1b..71409e38a5d2 100644 --- a/crates/typst-cli/tests/cnd.rs +++ b/crates/typst-cli/tests/cnd.rs @@ -191,3 +191,54 @@ fn normalize_uuids_keeps_references_aligned() { assert_eq!(normalize_uuids(a), normalize_uuids(b)); assert_ne!(normalize_uuids(a), normalize_uuids(c)); } + +#[test] +fn inputs_file_feeds_sys_inputs_and_input_flag_wins() { + let dir = tempfile::tempdir().unwrap(); + let main = + write(dir.path(), "main.typ", "#sys.inputs.at(\"a\") #sys.inputs.at(\"b\")"); + let inputs = write(dir.path(), "inputs.json", r#"{"a": "from-file", "b": "file-b"}"#); + let out = dir.path().join("out.cnd"); + run_ok( + exec() + .arg("compile") + .arg(&main) + .arg(&out) + .arg("--inputs-file") + .arg(&inputs) + .arg("--input") + .arg("b=from-flag"), + ); + let json = std::fs::read_to_string(&out).unwrap(); + assert!(json.contains("from-file"), "{json}"); + assert!(json.contains("from-flag") && !json.contains("file-b"), "{json}"); +} + +#[test] +fn inputs_file_works_for_every_format() { + let dir = tempfile::tempdir().unwrap(); + let main = write(dir.path(), "main.typ", "#sys.inputs.at(\"a\")"); + let inputs = write(dir.path(), "inputs.json", r#"{"a": "x"}"#); + run_ok(exec().arg("compile").arg(&main).arg("--inputs-file").arg(&inputs)); + assert!(dir.path().join("main.pdf").exists()); +} + +#[test] +fn bad_inputs_file_fails_with_its_path() { + let dir = tempfile::tempdir().unwrap(); + let main = write(dir.path(), "main.typ", "x"); + let inputs = write(dir.path(), "inputs.json", r#"{"a": 1}"#); + let out = exec() + .arg("compile") + .arg(&main) + .arg("--inputs-file") + .arg(&inputs) + .output() + .unwrap(); + assert!(!out.status.success()); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("inputs.json") && stderr.contains("must be a string"), + "{stderr}" + ); +} From e83e898fada0de2bb1b45cd6f705de82480abc3a Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 16:49:27 +0200 Subject: [PATCH 05/11] feat(cli): add --fallback-dir for missing project files A project file that does not exist is looked up by its file name alone in the fallback directory. A project file that exists always wins, a directory in the project stays an error, escaping paths fail before the lookup, and package files never fall back. Only regular files whose real path lies inside the fallback directory are served, so a symlink cannot lead out of it. `--deps` names the fallback file that was read. Co-Authored-By: Claude Opus 5.5 --- crates/typst-cli/src/args.rs | 10 ++ crates/typst-cli/src/watch.rs | 3 +- crates/typst-cli/src/world.rs | 158 +++++++++++++++++++- crates/typst-cli/tests/cnd.rs | 271 ++++++++++++++++++++++++++++++++++ 4 files changed, 437 insertions(+), 5 deletions(-) diff --git a/crates/typst-cli/src/args.rs b/crates/typst-cli/src/args.rs index 4d7ee08c76aa..8644ae16121e 100644 --- a/crates/typst-cli/src/args.rs +++ b/crates/typst-cli/src/args.rs @@ -470,6 +470,16 @@ pub struct WorldArgs { #[clap(long = "root", env = "TYPST_ROOT", value_name = "DIR")] pub root: Option, + /// A directory searched for project files that do not exist. + /// + /// When a file of the project is missing, a file with the same name (the + /// last path component only) in this directory is used instead. A file + /// that exists in the project always wins, and package files never fall + /// back. Only regular files that resolve to a location inside this + /// directory are served. + #[clap(long = "fallback-dir", value_name = "DIR")] + pub fallback_dir: Option, + /// Add a string key-value pair visible through `sys.inputs`. #[clap( short = 'i', diff --git a/crates/typst-cli/src/watch.rs b/crates/typst-cli/src/watch.rs index 87b4bb223f4c..e0cab8233c20 100644 --- a/crates/typst-cli/src/watch.rs +++ b/crates/typst-cli/src/watch.rs @@ -39,7 +39,8 @@ pub fn watch(command: &'static WatchCommand) -> HintedStrResult<()> { Ok(world) => break world, Err( ref err @ (WorldCreationError::InputNotFound(ref path) - | WorldCreationError::RootNotFound(ref path)), + | WorldCreationError::RootNotFound(ref path) + | WorldCreationError::FallbackDirNotFound(ref path)), ) => { watcher.update([path.clone()])?; Status::Error.print(&config).unwrap(); diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 2d0ff45eb9de..35e1ad5911ac 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -1,7 +1,8 @@ use std::error; use std::fmt; +use std::fs; use std::io::{self, Read}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::LazyLock; use ecow::{EcoString, eco_format}; @@ -206,6 +207,9 @@ static EMPTY_ID: LazyLock = LazyLock::new(|| { struct SystemFiles { main: FileId, project: FsRoot, + /// The canonical directory where missing project files are looked up by + /// file name. + fallback: Option, packages: SystemPackages, } @@ -255,16 +259,59 @@ impl SystemFiles { *EMPTY_ID }; + let fallback = + world_args.fallback_dir.as_deref().map(fallback_dir).transpose()?; + Ok(Self { main, project: FsRoot::new(root), + fallback, packages: crate::packages::system(&world_args.package), }) } - /// Resolves the file system path for the given `id`. + /// Resolves the file system path for the given `id`, including a file + /// served from the fallback directory. pub fn resolve(&self, id: FileId) -> FileResult { - self.root(id)?.resolve(id.vpath()) + let path = self.root(id)?.resolve(id.vpath())?; + Ok(self.fallback_file(id, &path).unwrap_or(path)) + } + + /// The fallback file that serves `id`, whose project path is `path`. + /// + /// Only a project file that does not exist falls back: a file that exists + /// wins, a directory stays an error, and an escaping path already failed + /// when its id was created. The file is looked up by its name alone and + /// is served only if it is a regular file whose real path lies inside the + /// fallback directory, so a symlink cannot lead out of it. + fn fallback_file(&self, id: FileId, path: &Path) -> Option { + let dir = self.fallback.as_deref()?; + if !matches!(id.root(), VirtualRoot::Project) + || id == *EMPTY_ID + || id == *STDIN_ID + { + return None; + } + // Any outcome but "not found" (an existing file or directory, a + // permission error, ...) keeps the project's own result. + match fs::metadata(path) { + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + _ => return None, + } + let name = id.vpath().file_name()?; + let mut components = Path::new(name).components(); + if !matches!( + (components.next(), components.next()), + (Some(Component::Normal(_)), None) + ) { + return None; + } + let candidate = dir.join(name); + if candidate.parent() != Some(dir) { + return None; + } + let real = candidate.canonicalize().ok()?; + (real.starts_with(dir) && real.is_file()).then_some(real) } /// Resolves the root in which the given file ID resides. @@ -283,11 +330,32 @@ impl FileLoader for SystemFiles { } else if id == *STDIN_ID { read_from_stdin().map(Bytes::new) } else { - self.root(id)?.load(id.vpath()) + let root = self.root(id)?; + let path = root.resolve(id.vpath())?; + if let Some(real) = self.fallback_file(id, &path) { + return fs::read(&real) + .map(Bytes::new) + .map_err(|err| FileError::from_io(err, &real)); + } + root.load(id.vpath()) } } } +/// Canonicalizes the `--fallback-dir` and checks that it is a directory. +fn fallback_dir(dir: &Path) -> Result { + let canonical = dir.canonicalize().map_err(|err| match err.kind() { + io::ErrorKind::NotFound => { + WorldCreationError::FallbackDirNotFound(dir.to_path_buf()) + } + _ => WorldCreationError::Io(err), + })?; + if !canonical.is_dir() { + return Err(WorldCreationError::FallbackDirNotADirectory(dir.to_path_buf())); + } + Ok(canonical) +} + /// Read from stdin. fn read_from_stdin() -> FileResult> { let mut buf = Vec::new(); @@ -309,6 +377,10 @@ pub enum WorldCreationError { InputMalformed(VirtualizeError), /// The root directory does not appear to exist. RootNotFound(PathBuf), + /// The fallback directory does not appear to exist. + FallbackDirNotFound(PathBuf), + /// The fallback directory is not a directory. + FallbackDirNotADirectory(PathBuf), /// The requested creation timestamp was invalid. InvalidTimestamp, /// The `--inputs-file` could not be used. @@ -338,6 +410,12 @@ impl fmt::Display for WorldCreationError { WorldCreationError::RootNotFound(path) => { write!(f, "root directory not found (searched at {})", path.display()) } + WorldCreationError::FallbackDirNotFound(path) => { + write!(f, "fallback directory not found (searched at {})", path.display()) + } + WorldCreationError::FallbackDirNotADirectory(path) => { + write!(f, "fallback directory is not a directory ({})", path.display()) + } WorldCreationError::InvalidTimestamp => { write!(f, "creation timestamp out of range") } @@ -377,3 +455,75 @@ impl From for typst::Feature { } } } + +#[cfg(test)] +mod tests { + use typst::syntax::package::PackageSpec; + use typst_kit::downloader::SystemDownloader; + use typst_kit::packages::{FsPackages, UniversePackages}; + + use super::*; + + /// A loader with the project in `project`, `fallback`, and local packages + /// in `packages`. + fn files(project: &Path, fallback: &Path, packages: &Path) -> SystemFiles { + SystemFiles { + main: *EMPTY_ID, + project: FsRoot::new(project.canonicalize().unwrap()), + fallback: Some(fallback.canonicalize().unwrap()), + packages: SystemPackages::from_parts( + Some(FsPackages::new(packages.to_path_buf())), + None, + // Not `crate::download::downloader()`: it parses the process + // arguments, which exits a test binary. + UniversePackages::new(SystemDownloader::new("test")), + ), + } + } + + fn id(root: VirtualRoot, path: &str) -> FileId { + RootedPath::new(root, VirtualPath::new(path).unwrap()).intern() + } + + #[test] + fn project_files_fall_back_and_resolve_to_the_fallback_file() { + let fallback = tempfile::tempdir().unwrap(); + fs::write(fallback.path().join("lib.typ"), "x").unwrap(); + let project = tempfile::tempdir().unwrap(); + let files = files(project.path(), fallback.path(), fallback.path()); + let id = id(VirtualRoot::Project, "nested/lib.typ"); + assert_eq!(files.load(id).unwrap().as_slice(), b"x"); + assert_eq!( + files.resolve(id).unwrap(), + fallback.path().join("lib.typ").canonicalize().unwrap() + ); + } + + #[test] + fn packages_never_fall_back() { + let fallback = tempfile::tempdir().unwrap(); + fs::write(fallback.path().join("lib.typ"), "x").unwrap(); + let packages = tempfile::tempdir().unwrap(); + // The package exists, but the requested file does not. + fs::create_dir_all(packages.path().join("local/pkg/0.1.0")).unwrap(); + let project = tempfile::tempdir().unwrap(); + let files = files(project.path(), fallback.path(), packages.path()); + let spec: PackageSpec = "@local/pkg:0.1.0".parse().unwrap(); + let id = id(VirtualRoot::Package(spec), "lib.typ"); + assert!(matches!(files.load(id), Err(FileError::NotFound(_)))); + assert!(!files.resolve(id).unwrap().starts_with(fallback.path())); + } + + #[test] + fn special_ids_never_fall_back() { + let fallback = tempfile::tempdir().unwrap(); + fs::write(fallback.path().join(""), "x").unwrap(); + fs::write(fallback.path().join(""), "x").unwrap(); + let project = tempfile::tempdir().unwrap(); + let files = files(project.path(), fallback.path(), fallback.path()); + assert!(files.load(*EMPTY_ID).unwrap().is_empty()); + let fallback = fallback.path().canonicalize().unwrap(); + assert!(!files.resolve(*EMPTY_ID).unwrap().starts_with(&fallback)); + assert!(!files.resolve(*STDIN_ID).unwrap().starts_with(&fallback)); + } +} diff --git a/crates/typst-cli/tests/cnd.rs b/crates/typst-cli/tests/cnd.rs index 71409e38a5d2..c776579d97cd 100644 --- a/crates/typst-cli/tests/cnd.rs +++ b/crates/typst-cli/tests/cnd.rs @@ -242,3 +242,274 @@ fn bad_inputs_file_fails_with_its_path() { "{stderr}" ); } + +/// A minimal image for `image(..)`. +const SVG_1X1: &str = r#""#; + +fn compile_with_fallback( + project: &Path, + main: &Path, + fallback: &Path, +) -> std::process::Output { + exec() + .arg("compile") + .arg(main) + .arg(project.join("out.pdf")) + .arg("--root") + .arg(project) + .arg("--fallback-dir") + .arg(fallback) + .arg("--deps") + .arg(project.join("deps.json")) + .arg("--deps-format") + .arg("json") + .output() + .unwrap() +} + +/// Compiles `main` to CND next to it, with a fallback directory. +fn compile_cnd_with_fallback(main: &Path, fallback: &Path) -> std::process::Output { + exec() + .arg("compile") + .arg(main) + .arg(main.with_file_name("out.cnd")) + .arg("--fallback-dir") + .arg(fallback) + .output() + .unwrap() +} + +fn deps_inputs(project: &Path) -> Vec { + let deps: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(project.join("deps.json")).unwrap(), + ) + .unwrap(); + deps["inputs"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap().to_owned()) + .collect() +} + +#[test] +fn missing_project_file_is_found_by_name_in_the_fallback_dir() { + let project = tempfile::tempdir().unwrap(); + let fallback = tempfile::tempdir().unwrap(); + write(fallback.path(), "logo.svg", SVG_1X1); + // From a nested file, with a directory prefix that does not exist, and + // as an absolute project path. + write( + project.path(), + "chapters/intro.typ", + "#image(\"logo.svg\")\n#image(\"assets/logo.svg\")\n#image(\"/logo.svg\")", + ); + let main = write(project.path(), "main.typ", "#include \"chapters/intro.typ\""); + let out = compile_with_fallback(project.path(), &main, fallback.path()); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let fallback_logo = fallback.path().join("logo.svg").canonicalize().unwrap(); + let inputs = deps_inputs(project.path()); + assert!( + inputs + .iter() + .any(|p| Path::new(p).canonicalize().ok().as_ref() == Some(&fallback_logo)), + "deps must name the fallback file: {inputs:?}", + ); + // The missing project paths are not reported as inputs. + assert!( + !inputs + .iter() + .any(|p| p.contains("assets") || p.ends_with("chapters/logo.svg")), + "{inputs:?}" + ); +} + +#[test] +fn project_file_wins_over_the_fallback() { + let project = tempfile::tempdir().unwrap(); + let fallback = tempfile::tempdir().unwrap(); + write(fallback.path(), "data.txt", "from-fallback"); + write(project.path(), "data.txt", "from-project"); + let main = write(project.path(), "main.typ", "#read(\"data.txt\")"); + let out = compile_cnd_with_fallback(&main, fallback.path()); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let json = std::fs::read_to_string(project.path().join("out.cnd")).unwrap(); + assert!(json.contains("from-project") && !json.contains("from-fallback"), "{json}"); +} + +#[test] +fn fallback_serves_source_files_too() { + let project = tempfile::tempdir().unwrap(); + let fallback = tempfile::tempdir().unwrap(); + write(fallback.path(), "shared.typ", "#let greeting = \"from-shared\""); + let main = write( + project.path(), + "main.typ", + "#import \"lib/shared.typ\": greeting\n#greeting", + ); + let out = compile_cnd_with_fallback(&main, fallback.path()); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let json = std::fs::read_to_string(project.path().join("out.cnd")).unwrap(); + assert!(json.contains("from-shared"), "{json}"); +} + +#[test] +fn escaping_paths_and_directories_never_fall_back() { + let outer = tempfile::tempdir().unwrap(); + let project = outer.path().join("project"); + let fallback = tempfile::tempdir().unwrap(); + write(fallback.path(), "secret.txt", "leaked"); + write(fallback.path(), "folder", "leaked: a file named like the project's folder"); + write(outer.path(), "secret.txt", "outside"); + std::fs::create_dir_all(project.join("folder")).unwrap(); + + for source in [ + "#read(\"../secret.txt\")", + "#read(\"/x/../../secret.txt\")", + "#read(\"folder\")", + "#read(\"folder/\")", + "#read(\"/\")", + ] { + let main = write(&project, "main.typ", source); + let out = compile_cnd_with_fallback(&main, fallback.path()); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success(), "{source} must fail: {stdout} {stderr}"); + assert!(!stderr.contains("leaked"), "{source}: {stderr}"); + assert!(!project.join("out.cnd").exists(), "{source}"); + } +} + +#[test] +fn directories_in_the_fallback_dir_are_not_served() { + let project = tempfile::tempdir().unwrap(); + let fallback = tempfile::tempdir().unwrap(); + std::fs::create_dir(fallback.path().join("data")).unwrap(); + let main = write(project.path(), "main.typ", "#read(\"data\")"); + let out = compile_cnd_with_fallback(&main, fallback.path()); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success()); + assert!(stderr.contains("file not found"), "{stderr}"); +} + +#[cfg(unix)] +#[test] +fn symlinks_leaving_the_fallback_dir_are_not_followed() { + let outer = tempfile::tempdir().unwrap(); + let fallback = outer.path().join("fallback"); + std::fs::create_dir(&fallback).unwrap(); + let secret = write(outer.path(), "secret.txt", "leaked"); + std::os::unix::fs::symlink(&secret, fallback.join("data.txt")).unwrap(); + std::os::unix::fs::symlink(outer.path(), fallback.join("up")).unwrap(); + let project = tempfile::tempdir().unwrap(); + + for source in ["#read(\"data.txt\")", "#read(\"up\")"] { + let main = write(project.path(), "main.typ", source); + let out = compile_cnd_with_fallback(&main, &fallback); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success(), "{source} must fail"); + assert!(stderr.contains("file not found"), "{source}: {stderr}"); + assert!(!stderr.contains("leaked"), "{source}: {stderr}"); + } +} + +#[cfg(unix)] +#[test] +fn symlinks_inside_the_fallback_dir_are_followed() { + let fallback = tempfile::tempdir().unwrap(); + write(fallback.path(), "real.txt", "from-fallback"); + std::os::unix::fs::symlink( + fallback.path().join("real.txt"), + fallback.path().join("data.txt"), + ) + .unwrap(); + let project = tempfile::tempdir().unwrap(); + let main = write(project.path(), "main.typ", "#read(\"data.txt\")"); + let out = compile_cnd_with_fallback(&main, fallback.path()); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let json = std::fs::read_to_string(project.path().join("out.cnd")).unwrap(); + assert!(json.contains("from-fallback"), "{json}"); +} + +#[test] +fn package_files_never_fall_back() { + let packages = tempfile::tempdir().unwrap(); + let pkg = packages.path().join("local/pkg/0.1.0"); + write( + &pkg, + "typst.toml", + "[package]\nname = \"pkg\"\nversion = \"0.1.0\"\nentrypoint = \"lib.typ\"\n", + ); + write(&pkg, "lib.typ", "#let value = read(\"data.txt\")"); + let fallback = tempfile::tempdir().unwrap(); + write(fallback.path(), "data.txt", "leaked"); + let project = tempfile::tempdir().unwrap(); + let main = + write(project.path(), "main.typ", "#import \"@local/pkg:0.1.0\": value\n#value"); + let compile = || { + exec() + .arg("compile") + .arg(&main) + .arg(project.path().join("out.cnd")) + .arg("--fallback-dir") + .arg(fallback.path()) + .arg("--package-path") + .arg(packages.path()) + .output() + .unwrap() + }; + + let out = compile(); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success(), "a package file must not fall back"); + assert!(stderr.contains("file not found"), "{stderr}"); + + // The same package compiles once its own file exists. + write(&pkg, "data.txt", "from-package"); + let out = compile(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); +} + +#[test] +fn missing_everywhere_keeps_the_usual_error() { + let project = tempfile::tempdir().unwrap(); + let fallback = tempfile::tempdir().unwrap(); + let main = write(project.path(), "main.typ", "#image(\"nowhere.png\")"); + let out = compile_with_fallback(project.path(), &main, fallback.path()); + assert!(!out.status.success()); + assert!(String::from_utf8_lossy(&out.stderr).contains("file not found")); +} + +#[test] +fn nonexistent_fallback_dir_is_reported() { + let project = tempfile::tempdir().unwrap(); + let main = write(project.path(), "main.typ", "x"); + let out = exec() + .arg("compile") + .arg(&main) + .arg("--fallback-dir") + .arg(project.path().join("nope")) + .output() + .unwrap(); + assert!(!out.status.success()); + assert!( + String::from_utf8_lossy(&out.stderr).contains("fallback directory not found") + ); +} + +#[test] +fn fallback_dir_must_be_a_directory() { + let project = tempfile::tempdir().unwrap(); + let main = write(project.path(), "main.typ", "x"); + let file = write(project.path(), "file.txt", "x"); + let out = exec() + .arg("compile") + .arg(&main) + .arg("--fallback-dir") + .arg(&file) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success()); + assert!(stderr.contains("fallback directory is not a directory"), "{stderr}"); +} From 165394276fb2ca053f5e8b654e1b8c0ad030b824 Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 17:00:35 +0200 Subject: [PATCH 06/11] feat(cli): report loaded font files in --deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream's --deps only lists files read through the file store; fonts served by FontStore were not reported. SystemWorld now tracks which font indices World::font served since the last reset and exposes them as font_dependencies() — sorted, deduplicated file paths, embedded fonts (no file) omitted. Every --deps format (json, make, zero) includes them via relative_dependencies(); watch.rs is untouched, so typst watch still does not watch font files. Co-Authored-By: Claude Sonnet 5 --- crates/typst-cli/src/deps.rs | 4 ++- crates/typst-cli/src/world.rs | 25 ++++++++++++-- crates/typst-cli/tests/cnd.rs | 61 +++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/crates/typst-cli/src/deps.rs b/crates/typst-cli/src/deps.rs index c76ea70c2181..1fc6e99c0e5f 100644 --- a/crates/typst-cli/src/deps.rs +++ b/crates/typst-cli/src/deps.rs @@ -170,7 +170,9 @@ fn relative_dependencies( let current_dir = std::env::current_dir()?; let relative_root = pathdiff::diff_paths(&root, ¤t_dir).unwrap_or_else(|| root.clone()); - Ok(world.dependencies().map(move |dependency| { + let fonts = world.font_dependencies(); + let files: Vec = world.dependencies().collect(); + Ok(files.into_iter().chain(fonts).map(move |dependency| { dependency .strip_prefix(&root) .map_or_else(|_| dependency.clone(), |x| relative_root.join(x)) diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 35e1ad5911ac..2d7d52b57a4d 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -1,9 +1,11 @@ +use std::any::Any; +use std::collections::BTreeSet; use std::error; use std::fmt; use std::fs; use std::io::{self, Read}; use std::path::{Component, Path, PathBuf}; -use std::sync::LazyLock; +use std::sync::{LazyLock, Mutex}; use ecow::{EcoString, eco_format}; use typst::diag::{FileError, FileResult}; @@ -17,7 +19,7 @@ use typst::{Library, LibraryExt, World}; use typst_kit::datetime::Time; use typst_kit::diagnostics::DiagnosticWorld; use typst_kit::files::{FileLoader, FileStore, FsRoot}; -use typst_kit::fonts::FontStore; +use typst_kit::fonts::{FontPath, FontStore}; use typst_kit::packages::SystemPackages; use crate::args::{Feature, Input, OutputFormat, ProcessArgs, WorldArgs}; @@ -36,6 +38,8 @@ pub struct SystemWorld { /// always the same within one compilation. /// Reset between compilations if not [`Time::Fixed`]. now: Time, + /// Indices of the fonts loaded since the last reset, for `--deps`. + accessed_fonts: Mutex>, } impl SystemWorld { @@ -101,6 +105,7 @@ impl SystemWorld { })), files: FileStore::new(SystemFiles::new(input, world_args)?), now, + accessed_fonts: Mutex::new(BTreeSet::new()), }) } @@ -124,6 +129,7 @@ impl SystemWorld { pub fn reset(&mut self) { self.files.reset(); self.now.reset(); + self.accessed_fonts.get_mut().unwrap().clear(); } /// Forcibly scan fonts instead of doing it lazily upon the first access. @@ -132,6 +138,20 @@ impl SystemWorld { pub fn scan_fonts(&mut self) { LazyLock::force(&self.fonts); } + + /// Font files loaded by the last compilation, sorted and deduplicated. + /// Fonts without a file (embedded ones) are left out. This is every font + /// the layout loaded, which can include fonts tried for glyph fallback. + pub fn font_dependencies(&self) -> Vec { + let accessed = self.accessed_fonts.lock().unwrap(); + let paths: BTreeSet = accessed + .iter() + .filter_map(|&index| self.fonts.source(index)) + .filter_map(|source| (source as &dyn Any).downcast_ref::()) + .map(|font| font.path.clone()) + .collect(); + paths.into_iter().collect() + } } impl World for SystemWorld { @@ -156,6 +176,7 @@ impl World for SystemWorld { } fn font(&self, index: usize) -> Option { + self.accessed_fonts.lock().unwrap().insert(index); self.fonts.font(index) } diff --git a/crates/typst-cli/tests/cnd.rs b/crates/typst-cli/tests/cnd.rs index c776579d97cd..e4a4b71b4bfd 100644 --- a/crates/typst-cli/tests/cnd.rs +++ b/crates/typst-cli/tests/cnd.rs @@ -513,3 +513,64 @@ fn fallback_dir_must_be_a_directory() { assert!(!out.status.success()); assert!(stderr.contains("fallback directory is not a directory"), "{stderr}"); } + +#[test] +fn deps_lists_font_files_the_compile_loaded() { + let project = tempfile::tempdir().unwrap(); + let fonts = tempfile::tempdir().unwrap(); + let data = typst_dev_assets::fonts().next().unwrap(); + let family = typst::text::Font::new(typst::foundations::Bytes::new(data), 0) + .unwrap() + .info() + .family + .clone(); + write(fonts.path(), "custom.ttf", data); + write(fonts.path(), "unused.ttf", data); // same family twice: only one is loaded + let main = write( + project.path(), + "main.typ", + format!("#set text(font: \"{family}\")\nHello"), + ); + let deps = project.path().join("deps.json"); + run_ok( + exec() + .arg("compile") + .arg(&main) + .arg(project.path().join("out.cnd")) + .arg("--ignore-system-fonts") + .arg("--font-path") + .arg(fonts.path()) + .arg("--deps") + .arg(&deps) + .arg("--deps-format") + .arg("json"), + ); + let inputs = deps_inputs(project.path()); + let font_inputs: Vec<&String> = + inputs.iter().filter(|p| p.ends_with(".ttf")).collect(); + assert_eq!(font_inputs.len(), 1, "exactly one font file loaded: {inputs:?}"); + assert!(inputs.iter().any(|p| p.ends_with("main.typ"))); +} + +#[test] +fn deps_without_custom_fonts_lists_no_font_file() { + let project = tempfile::tempdir().unwrap(); + let main = write(project.path(), "main.typ", "Hello"); + run_ok( + exec() + .arg("compile") + .arg(&main) + .arg(project.path().join("out.cnd")) + .arg("--ignore-system-fonts") + .arg("--deps") + .arg(project.path().join("deps.json")) + .arg("--deps-format") + .arg("json"), + ); + // Embedded fonts have no file. + assert!( + deps_inputs(project.path()) + .iter() + .all(|p| !p.ends_with(".ttf") && !p.ends_with(".otf")) + ); +} From 4664fa04ea12775520904dabd997c2a5b8812776 Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 19:23:45 +0200 Subject: [PATCH 07/11] docs: typst compile --format cnd is the CND entry point Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 10 +++++++++- crates/typst-cnd/src/bin/typst-cnd.rs | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 7e2dfcb55848..c8dc4d85b624 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -19,7 +19,7 @@ This directory is a **local clone of the Typst compiler** (v0.14.x workspace). I ```plain text Typst source (.typ) - │ typst-cnd (Rust, this workspace) + │ typst compile --format cnd (typst-cli + typst-cnd) ▼ CND Manifest (JSON) │ cnd-engine (Python) @@ -36,6 +36,14 @@ flowchart LR E --> F["Qdrant"] ``` +`typst-cnd` is the exporter library. The supported entry point is +`typst compile --format cnd` (or an output path ending in `.cnd`), +which brings every standard CLI option (`--root`, `--font-path`, +`--ignore-system-fonts`, `--deps`) plus two generic additions: +`--inputs-file` (a JSON object of string values for `sys.inputs`) and +`--fallback-dir` (missing project files are looked up by file name there). +The standalone `typst-cnd` binary remains for existing callers. + ## Strategic context (Notion, June 2026) Two strategies were documented in the [CND Notion wiki](https://www.notion.so/maxanox/36cc61707d5f800da28ac1f757e03270): diff --git a/crates/typst-cnd/src/bin/typst-cnd.rs b/crates/typst-cnd/src/bin/typst-cnd.rs index 0138f0d52cc0..a0880d3739f3 100644 --- a/crates/typst-cnd/src/bin/typst-cnd.rs +++ b/crates/typst-cnd/src/bin/typst-cnd.rs @@ -11,7 +11,10 @@ use typst_cnd::{CndDocument, cnd_from_document, cnd_to_json, world}; use typst_syntax::Span; #[derive(Parser)] -#[command(name = "typst-cnd", about = "Compile Typst sources into CND JSON")] +#[command( + name = "typst-cnd", + about = "Compile Typst sources into CND JSON (also available as `typst compile --format cnd`)" +)] struct Cli { #[command(subcommand)] command: Commands, From f6204c1d605bea74178eb05e69c0f6ef2b53b99e Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 19:44:02 +0200 Subject: [PATCH 08/11] fix(cli): report font files in --deps by their canonical path Font files were reported by the path the font search found them at, so a --font-path given through a symlink or as a relative path was reported differently from a file dependency in the same directory, and two spellings of one file were listed twice. They are now canonicalized like file dependencies, then made relative to the root when inside it. The integration tests of `typst compile --format cnd` also clear every environment variable the CLI reads as a flag default, so a TYPST_ROOT or TYPST_FONT_PATHS in the environment cannot change their outcome. Co-Authored-By: Claude Opus 5.5 --- crates/typst-cli/src/world.rs | 10 ++-- crates/typst-cli/tests/cnd.rs | 87 ++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 2d7d52b57a4d..b25a4145638b 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -43,7 +43,8 @@ pub struct SystemWorld { } impl SystemWorld { - /// Creates a new system world for any format but CND. + /// Creates a new system world with the default (non-CND) standard + /// library, as used by every format but CND and by `query` and `eval`. pub fn new( input: Option<&Input>, world_args: &'static WorldArgs, @@ -139,7 +140,8 @@ impl SystemWorld { LazyLock::force(&self.fonts); } - /// Font files loaded by the last compilation, sorted and deduplicated. + /// Font files loaded by the last compilation, canonicalized (when + /// possible), sorted and deduplicated. /// Fonts without a file (embedded ones) are left out. This is every font /// the layout loaded, which can include fonts tried for glyph fallback. pub fn font_dependencies(&self) -> Vec { @@ -148,7 +150,9 @@ impl SystemWorld { .iter() .filter_map(|&index| self.fonts.source(index)) .filter_map(|source| (source as &dyn Any).downcast_ref::()) - .map(|font| font.path.clone()) + // Canonical, like file dependencies, so two spellings of one + // file (a symlink, a relative path) are reported once. + .map(|font| font.path.canonicalize().unwrap_or_else(|_| font.path.clone())) .collect(); paths.into_iter().collect() } diff --git a/crates/typst-cli/tests/cnd.rs b/crates/typst-cli/tests/cnd.rs index e4a4b71b4bfd..4df7cce015b3 100644 --- a/crates/typst-cli/tests/cnd.rs +++ b/crates/typst-cli/tests/cnd.rs @@ -5,8 +5,27 @@ use std::process::Command; use typst_cnd::Cnd; +/// Environment variables the CLI reads as defaults for its flags. A test +/// must not depend on the environment it runs in. +const CLI_ENV: &[&str] = &[ + "TYPST_CERT", + "TYPST_ROOT", + "SOURCE_DATE_EPOCH", + "TYPST_FEATURES", + "TYPST_DIAGNOSTIC_FORMAT", + "TYPST_PACKAGE_PATH", + "TYPST_PACKAGE_CACHE_PATH", + "TYPST_FONT_PATHS", + "TYPST_IGNORE_SYSTEM_FONTS", + "TYPST_IGNORE_EMBEDDED_FONTS", +]; + fn exec() -> Command { - Command::new(env!("CARGO_BIN_EXE_typst")) + let mut cmd = Command::new(env!("CARGO_BIN_EXE_typst")); + for var in CLI_ENV { + cmd.env_remove(var); + } + cmd } fn write(dir: &Path, rel: &str, data: impl AsRef<[u8]>) -> PathBuf { @@ -574,3 +593,69 @@ fn deps_without_custom_fonts_lists_no_font_file() { .all(|p| !p.ends_with(".ttf") && !p.ends_with(".otf")) ); } + +/// A font directory reached through a symlink or a relative path is reported +/// by its canonical path, like a fallback file from the same directory: +/// relative to `--root` when inside it, absolute otherwise. +#[cfg(unix)] +#[test] +fn deps_report_font_files_by_their_canonical_path() { + let outer = tempfile::tempdir().unwrap(); + let outer = outer.path().canonicalize().unwrap(); + let project = outer.join("project"); + let data = typst_dev_assets::fonts().next().unwrap(); + let family = typst::text::Font::new(typst::foundations::Bytes::new(data), 0) + .unwrap() + .info() + .family + .clone(); + for dir in [project.join("fonts"), outer.join("library/fonts")] { + write(&dir, "custom.ttf", data); + write(&dir, "logo.svg", SVG_1X1); + } + std::os::unix::fs::symlink(outer.join("library"), outer.join("link")).unwrap(); + let main = write( + &project, + "main.typ", + format!("#set text(font: \"{family}\")\nHello\n#image(\"logo.svg\")"), + ); + + let absolute = |p: PathBuf| p.to_str().unwrap().to_owned(); + let cases = [ + // Outside the root, through a symlinked directory: absolute, canonical. + ( + outer.join("link/fonts"), + absolute(outer.join("library/fonts/custom.ttf")), + absolute(outer.join("library/fonts/logo.svg")), + ), + // Inside the root, as a relative path: relative to the root. + ( + PathBuf::from("../project/fonts"), + "fonts/custom.ttf".to_owned(), + "fonts/logo.svg".to_owned(), + ), + ]; + for (dir, font, logo) in cases { + run_ok( + exec() + .current_dir(&project) + .arg("compile") + .arg(&main) + .arg(project.join("out.cnd")) + .arg("--root") + .arg(&project) + .arg("--ignore-system-fonts") + .arg("--font-path") + .arg(&dir) + .arg("--fallback-dir") + .arg(&dir) + .arg("--deps") + .arg(project.join("deps.json")) + .arg("--deps-format") + .arg("json"), + ); + let inputs = deps_inputs(&project); + assert!(inputs.contains(&logo), "{}: {logo} in {inputs:?}", dir.display()); + assert!(inputs.contains(&font), "{}: {font} in {inputs:?}", dir.display()); + } +} From dcef9c5f30cbae2a17b45d31c8973b81794ffe40 Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 19:44:12 +0200 Subject: [PATCH 09/11] docs: name the CLI entry point in the pipeline diagram Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTEXT.md b/CONTEXT.md index c8dc4d85b624..09117c2016eb 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -29,7 +29,7 @@ Chunks → DBR records → embedding → Qdrant ```mermaid flowchart LR - A[".typ"] --> B["typst-cnd"] + A[".typ"] --> B["typst compile --format cnd"] B --> C["manifest.json"] C --> D["cnd-engine"] D --> E["DBR"] From b38516a6acb98e74570dbb1445e3ec27ccb57357 Mon Sep 17 00:00:00 2001 From: Maxanox Date: Fri, 25 Sep 2026 20:46:18 +0200 Subject: [PATCH 10/11] docs: rewrite CONTEXT.md around the fork and the CND standard Drops stale planning notes, local paths and links to private notes; keeps the hook points, the pipeline and the contributor rules. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 198 +++++------------- .../typst-library/src/model/bibliography.rs | 6 +- 2 files changed, 54 insertions(+), 150 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 09117c2016eb..688c8967bbe4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,39 +1,35 @@ # typst + typst-cnd — Project Context -Reference map for contributors and AI agents working on the **typst-cnd** compiler inside this Typst workspace. +Reference map for contributors and AI agents working on the **typst-cnd** +exporter inside this fork of the Typst compiler. -## What this workspace is +## What this fork is -This directory is a **local clone of the Typst compiler** (v0.14.x workspace). It is the home of **`typst-cnd`**: a new exporter crate that compiles `.typ` sources into **CND Manifest JSON** for the Cosmyc / ctxnative pipeline. +A fork of the Typst compiler that adds **`typst-cnd`**, an exporter crate that +compiles `.typ` sources into **CND** JSON, a structured document format defined +by the CND standard and its SDK (`cnd-sdk`). Everything else is upstream Typst, +merged regularly. | Piece | Location | Role | |---|---|---| | Typst compiler (upstream) | `crates/typst*`, `crates/typst-cli`, … | General-purpose typesetting | -| **typst-cnd** (to build) | `crates/typst-cnd/` | Typst → CND manifest JSON | -| CND consumer (separate repo) | `/home/maxence/ctxnative/cnd-engine` | Load manifest, chunk, DBR, Qdrant | -| Cosmyc product context | Notion wiki **CND** | Spec, engine, product vision | +| **typst-cnd** | `crates/typst-cnd/` | Typst → CND JSON exporter | +| CND standard and SDK | `cnd-sdk` (separate repository) | Format definition, validation, consumers | -**Do not** put CND emission logic in `cnd-engine` (Python). **Do not** patch `typst-layout` / `typst-eval` unless a public API gap forces it. Add a peer crate like `typst-html` and `typst-pdf`. +**Do not** patch `typst-layout` / `typst-eval` unless a public API gap forces +it. Exporter logic lives in `crates/typst-cnd/`, a peer of `typst-html` and +`typst-pdf`; `typst-cli` only wires it in as an output format. -## End-to-end pipeline +## Entry point ```plain text Typst source (.typ) │ typst compile --format cnd (typst-cli + typst-cnd) ▼ -CND Manifest (JSON) - │ cnd-engine (Python) +CND (JSON) + │ any CND consumer (see cnd-sdk) ▼ -Chunks → DBR records → embedding → Qdrant -``` - -```mermaid -flowchart LR - A[".typ"] --> B["typst compile --format cnd"] - B --> C["manifest.json"] - C --> D["cnd-engine"] - D --> E["DBR"] - E --> F["Qdrant"] +Validation, chunking, indexing, rendering, … ``` `typst-cnd` is the exporter library. The supported entry point is @@ -44,28 +40,13 @@ which brings every standard CLI option (`--root`, `--font-path`, `--fallback-dir` (missing project files are looked up by file name there). The standalone `typst-cnd` binary remains for existing callers. -## Strategic context (Notion, June 2026) - -Two strategies were documented in the [CND Notion wiki](https://www.notion.so/maxanox/36cc61707d5f800da28ac1f757e03270): - -| Plan | Compilateur Typst | Format interne | -|---|---|---| -| Pivot DocLang | `typst-doclang` → `.dclg.xml` | DocLang pivot | -| **Alternative CND v2** (favored for this work) | **`typst-cnd`** → manifest JSON | CND v2 canonique | - -**Current direction for this repo:** implement **`typst-cnd`** as the premium Typst on-ramp. DocLang remains a possible **interop boundary** for legacy PDF/DOCX ingestion — not the hot path for native Typst. - -Key pages: -- [CND Standard](https://app.notion.com/p/37dc61707d5f8082b495d1fad337d087) — manifest schema, flags, refs -- [CND Engine](https://app.notion.com/p/37dc61707d5f80228ebada1dc7ace089) — what downstream expects -- [Alternative CND v2 + DocLang interchange](https://app.notion.com/p/37ec61707d5f81068ec8ea894a399bbc) — why typst-cnd stays central -- [Plan de transition DocLang](https://app.notion.com/p/37ec61707d5f815a9935c79c429b46bf) — context only; superseded for Typst path if CND v2 wins - ## Where typst-cnd hooks into Typst -CND nodes are **not** built from the syntax AST (`typst_syntax::SyntaxNode`). They come from the **typed content tree after evaluation and realization** — the same layer `typst-html` uses. +CND nodes are **not** built from the syntax AST (`typst_syntax::SyntaxNode`). +They come from the **typed content tree after evaluation and realization** — +the same layer `typst-html` uses. -Reference implementation to study: +Reference implementations to study: ```plain text crates/typst-html/src/document.rs → html_document(), realize() @@ -77,13 +58,13 @@ crates/typst-cli/src/compile.rs → World + typst::compile:: ### Pipeline inside typst-cnd ```plain text -1. eval(main) → Content (HeadingElem, ParElem, TableElem, …) -2. realize(Document) → structured element pairs (see typst-html) -3. walk / convert → CndNode tree (heading children, tables, …) +1. eval(main) → Content (HeadingElem, ParElem, TableElem, …) +2. realize(Document) → structured element pairs (see typst-html) +3. walk / convert → CndNode tree (heading children, tables, …) 4. compile:: → layout + stable Introspector -5. join locations → NodeLocation (page, span, …) per CND node -6. resolve refs → refs_to / refs_from as NodeRef { id, label? } -7. serialize → JSON manifest +5. join locations → NodeLocation (starting page) per CND node +6. resolve refs → refs_to / refs_from as NodeRef { id, label? } +7. serialize → CND JSON ``` | CND field | Typst source | @@ -91,119 +72,42 @@ crates/typst-cli/src/compile.rs → World + typst::compile:: | `heading`, `paragraph`, `table` | `HeadingElem`, `ParElem`, `TableElem` after `realize` | | `label` | Element labels (`