From f4cbd077c561ca5a96e8b506df722b793dfc0efc Mon Sep 17 00:00:00 2001 From: albedosehen Date: Sun, 4 Oct 2026 09:23:49 -0800 Subject: [PATCH 1/4] Default to the MonotoneGraphemes cluster level, as HarfBuzz does ClusterLevel::default(), and so a new Buffer, was MonotoneCharacters, where HarfBuzz and the C API default to MONOTONE_GRAPHEMES. A combining mark, the second regional indicator of a flag, a ZWJ emoji sequence and the other characters a grapheme merges each kept a cluster of their own under the defaults: 73 of the 554 consumer differential cases had other clusters than HarfBuzz 14.5.0 there. The default is now MonotoneGraphemes, and all 554 match with clusters compared. Callers who want a cluster per character set ClusterLevel::MonotoneCharacters. The C API already defaulted to HarfBuzz's level and is unchanged. The tests that read clusters at the default level now expect HarfBuzz's default-level output. --- docs/STABILITY.md | 6 +++--- src/buffer/flags.rs | 36 ++++++++++++++++++++--------------- src/ot/mongolian.rs | 3 +-- src/ot/tibetan.rs | 3 +-- src/ot/use_shaper/scripts.rs | 2 +- src/shape/tests.rs | 9 +++++++-- tests/cluster_level_parity.rs | 12 +++++++----- tests/lookup_engine_parity.rs | 18 ++++++++++-------- 8 files changed, 51 insertions(+), 38 deletions(-) diff --git a/docs/STABILITY.md b/docs/STABILITY.md index 4fa533b2..b7989fcb 100644 --- a/docs/STABILITY.md +++ b/docs/STABILITY.md @@ -86,9 +86,9 @@ signatures only change in a major version. `Buffer::{set_insert_dotted_circle, insert_dotted_circle}` pair that 0.22.0 development builds had. - `ClusterLevel` and `Buffer::{set_cluster_level, cluster_level}`: HarfBuzz's four - cluster levels. A Rust `Buffer` defaults to `MonotoneCharacters`; the C API - defaults to HarfBuzz's `MONOTONE_GRAPHEMES`. Both settings survive - `Buffer::clear`. + cluster levels. A Rust `Buffer` and the C API both default to HarfBuzz's + `MonotoneGraphemes` (`MONOTONE_GRAPHEMES`); before 0.24.0 a Rust `Buffer` + defaulted to `MonotoneCharacters`. Both settings survive `Buffer::clear`. `sigilbuzz-render`: diff --git a/src/buffer/flags.rs b/src/buffer/flags.rs index 9752fa01..af9a22ea 100644 --- a/src/buffer/flags.rs +++ b/src/buffer/flags.rs @@ -250,10 +250,11 @@ impl SubAssign for BufferFlags { /// clusters and a reordered glyph keeps its own offset, so clusters /// can come out of order. /// -/// HarfBuzz defaults to [`Self::MonotoneGraphemes`] (the C API does -/// too). A Rust [`Buffer`] defaults to [`Self::MonotoneCharacters`], -/// the level closest to what sigilbuzz produced before it supported -/// cluster levels. +/// The default is [`Self::MonotoneGraphemes`], as in HarfBuzz and the +/// C API. Before 0.24.0 a Rust [`Buffer`] defaulted to +/// [`Self::MonotoneCharacters`], which gives a combining mark (and the +/// other characters a grapheme merges) a cluster of its own; call +/// [`Buffer::set_cluster_level`] with it to keep those clusters. /// /// # Examples /// @@ -261,20 +262,22 @@ impl SubAssign for BufferFlags { /// use sigilbuzz::{Buffer, ClusterLevel}; /// /// let mut buffer = Buffer::new(); -/// assert_eq!(buffer.cluster_level(), ClusterLevel::MonotoneCharacters); -/// buffer.set_cluster_level(ClusterLevel::MonotoneGraphemes); +/// assert_eq!(buffer.cluster_level(), ClusterLevel::MonotoneGraphemes); /// assert!(buffer.cluster_level().is_monotone()); /// assert!(buffer.cluster_level().is_graphemes()); +/// buffer.set_cluster_level(ClusterLevel::MonotoneCharacters); +/// assert!(buffer.cluster_level().is_characters()); /// ``` #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] pub enum ClusterLevel { /// Characters merge into their grapheme, and clusters stay in /// order (`HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES`, HarfBuzz's - /// default). + /// default and sigilbuzz's). + #[default] MonotoneGraphemes, /// Every character starts with its own cluster, and clusters stay - /// in order (`HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS`). - #[default] + /// in order (`HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS`). The + /// default of a Rust [`Buffer`] before 0.24.0. MonotoneCharacters, /// Every character keeps its own cluster, in whatever order /// shaping leaves them (`HB_BUFFER_CLUSTER_LEVEL_CHARACTERS`). @@ -336,7 +339,8 @@ impl Buffer { } /// The cluster level set with [`Self::set_cluster_level`]; - /// [`ClusterLevel::MonotoneCharacters`] until then. + /// [`ClusterLevel::MonotoneGraphemes`], HarfBuzz's default, until + /// then. #[must_use] pub const fn cluster_level(&self) -> ClusterLevel { self.cluster_level @@ -357,14 +361,15 @@ impl Buffer { /// let mut buffer = Buffer::new(); /// buffer.push_str("x\u{0301}"); /// - /// // The combining acute keeps its own cluster ... + /// // The default grapheme level merges the combining acute into + /// // the cluster of its base ... /// let clusters = |b: &Buffer| -> Vec { /// shape(&font, b, &[]).unwrap().glyphs.iter().map(|g| g.cluster).collect() /// }; - /// assert_eq!(clusters(&buffer), [0, 1]); - /// // ... until the grapheme levels merge it into its base. - /// buffer.set_cluster_level(ClusterLevel::MonotoneGraphemes); /// assert_eq!(clusters(&buffer), [0, 0]); + /// // ... and the character levels give it its own. + /// buffer.set_cluster_level(ClusterLevel::MonotoneCharacters); + /// assert_eq!(clusters(&buffer), [0, 1]); /// # Ok::<(), sigilbuzz::Error>(()) /// ``` pub fn set_cluster_level(&mut self, level: ClusterLevel) { @@ -468,7 +473,8 @@ mod tests { fn flags_and_level_survive_clear() { let mut b = Buffer::new(); assert_eq!(b.flags(), BufferFlags::DEFAULT); - assert_eq!(b.cluster_level(), ClusterLevel::MonotoneCharacters); + assert_eq!(b.cluster_level(), ClusterLevel::MonotoneGraphemes); + assert_eq!(ClusterLevel::default(), ClusterLevel::MonotoneGraphemes); b.set_flags(BufferFlags::BOT | BufferFlags::REMOVE_DEFAULT_IGNORABLES); b.set_cluster_level(ClusterLevel::Characters); b.push_str("abc"); diff --git a/src/ot/mongolian.rs b/src/ot/mongolian.rs index e83f0efb..c65fbf96 100644 --- a/src/ot/mongolian.rs +++ b/src/ot/mongolian.rs @@ -109,8 +109,7 @@ pub fn assign_mongolian_forms_in_context( /// /// `codepoints` and `glyphs` start 1:1 (a glyph per codepoint, post /// cmap). After the call `glyphs` may have shrunk through ligature -/// collapse. Clusters merge at the monotone characters level, the -/// default of a Rust [`crate::Buffer`]. +/// collapse. Clusters merge at the monotone characters level. pub fn shape_mongolian( gsub: Option<&Gsub<'_>>, gdef: Option<&Gdef<'_>>, diff --git a/src/ot/tibetan.rs b/src/ot/tibetan.rs index be55e6e9..e98ef8ba 100644 --- a/src/ot/tibetan.rs +++ b/src/ot/tibetan.rs @@ -42,8 +42,7 @@ pub const fn is_tibetan(ch: char) -> bool { /// Engine, as HarfBuzz does, every GSUB feature included. /// `codepoints` and `glyphs` are 1:1 on entry. After the call `glyphs` /// may have shrunk (ligature) or grown (multiple substitution). -/// Clusters merge at the monotone characters level, the default of a -/// Rust [`crate::Buffer`]. +/// Clusters merge at the monotone characters level. pub fn shape_tibetan( gsub: Option<&Gsub<'_>>, gdef: Option<&Gdef<'_>>, diff --git a/src/ot/use_shaper/scripts.rs b/src/ot/use_shaper/scripts.rs index bcb64657..af8dbe5e 100644 --- a/src/ot/use_shaper/scripts.rs +++ b/src/ot/use_shaper/scripts.rs @@ -87,7 +87,7 @@ fn merge_jamo_syllables(codepoints: &[char], glyphs: &mut [Glyph], level: Cluste /// `nko ` script tags, whose `isol`, `init`, `medi`, and `fina` follow /// the Arabic-style joining forms of the letters, as in HarfBuzz /// (`setup_masks_arabic_plan`). Clusters merge at the monotone -/// characters level, the default of a Rust [`crate::Buffer`]. +/// characters level. pub fn shape_nko( gsub: Option<&Gsub<'_>>, gdef: Option<&Gdef<'_>>, diff --git a/src/shape/tests.rs b/src/shape/tests.rs index e7c79b3c..80e6a685 100644 --- a/src/shape/tests.rs +++ b/src/shape/tests.rs @@ -214,8 +214,9 @@ fn feature_slice_is_accepted_but_ignored_today() { fn decomposed_input_the_font_cannot_compose_keeps_both_characters() { // The test font maps neither 'e', the combining acute, nor the // precomposed 'é': normalization only recomposes into a composite - // the font maps, so both characters stay, each on .notdef, and - // each keeps its own cluster. + // the font maps, so both characters stay, each on .notdef. The + // default grapheme level puts the acute in the cluster of its + // base; the character levels give it its own. let data = build_shapeable_font(); let blob = Blob::new(&data); let face = Face::parse(&blob, 0).unwrap(); @@ -226,6 +227,10 @@ fn decomposed_input_the_font_cannot_compose_keeps_both_characters() { let shaped = shape(&font, &buffer, &[]).unwrap(); assert_eq!(shaped.len(), 2); assert_eq!(shaped.glyphs[0].glyph_id, 0); + assert_eq!(shaped.glyphs[1].cluster, 0); + buffer.set_cluster_level(crate::ClusterLevel::MonotoneCharacters); + let shaped = shape(&font, &buffer, &[]).unwrap(); + assert_eq!(shaped.len(), 2); assert_eq!(shaped.glyphs[1].cluster, 1); } diff --git a/tests/cluster_level_parity.rs b/tests/cluster_level_parity.rs index 7fb14f61..34fa6bf2 100644 --- a/tests/cluster_level_parity.rs +++ b/tests/cluster_level_parity.rs @@ -337,9 +337,10 @@ fn graphemes_level_forms_graphemes_without_monotone_merges() { } #[test] -fn the_default_level_is_monotone_characters() { - // A base and a mark that do not compose (normalization would fold - // "e\u{0301}" into one glyph at every level). +fn the_default_level_is_monotone_graphemes() { + // HarfBuzz's default. A base and a mark that do not compose + // (normalization would fold "e\u{0301}" into one glyph at every + // level): the mark joins the base's cluster. let text = "q\u{0301}"; let blob = Blob::new(OPEN_SANS); let font = Font::new(Face::parse(&blob, 0).expect("face"), 1000.0); @@ -357,8 +358,9 @@ fn the_default_level_is_monotone_characters() { OPEN_SANS, text, Direction::Ltr, - ClusterLevel::MonotoneCharacters + ClusterLevel::MonotoneGraphemes ) ); - assert_eq!(got, [0, 1]); + assert_eq!(got, [0, 0]); + assert_eq!(ClusterLevel::default(), ClusterLevel::MonotoneGraphemes); } diff --git a/tests/lookup_engine_parity.rs b/tests/lookup_engine_parity.rs index bca9b94a..298ddfb6 100644 --- a/tests/lookup_engine_parity.rs +++ b/tests/lookup_engine_parity.rs @@ -91,13 +91,15 @@ fn a_cgj_that_blocked_mark_reordering_stays_hidden_from_gsub() { // // Acute (230) then cedilla (202): the CGJ kept them from being // reordered, so it stays hidden and the acute keeps its own form. + // At the default grapheme level the marks and the CGJ share the + // cluster of the base. assert_eq!( marks(RUBIK, "f\u{0301}\u{034F}\u{0327}"), [ (162, 0, 0, 0), - (1126, 1, -280, 190), - (928, 3, 0, 0), - (1137, 5, -340, 0) + (1126, 0, -280, 190), + (928, 0, 0, 0), + (1137, 0, -340, 0) ] ); // Cedilla then acute were in order: the CGJ is skipped and the @@ -106,9 +108,9 @@ fn a_cgj_that_blocked_mark_reordering_stays_hidden_from_gsub() { marks(RUBIK, "f\u{0327}\u{034F}\u{0301}"), [ (162, 0, 0, 0), - (1154, 1, -340, 0), - (928, 3, 0, 0), - (1145, 5, -308, 10) + (1154, 0, -340, 0), + (928, 0, 0, 0), + (1145, 0, -308, 10) ] ); // Before a base the CGJ blocks nothing either: "f", CGJ, "i" @@ -143,10 +145,10 @@ fn a_gpos_context_subtable_that_does_not_match_leaves_the_next_its_turn() { assert_eq!( got, [ - (97, 12, 0, 134), + (97, 10, 0, 134), (62, 10, 661, 0), (4263, 8, 261, 0), - (97, 6, 0, -9), + (97, 4, 0, -9), (4173, 4, 343, 0), (4366, 2, 336, 0), (4330, 0, 674, 0), From ec7652de89aa660a06e11ebd84e5327686522fc1 Mon Sep 17 00:00:00 2001 From: albedosehen Date: Sun, 4 Oct 2026 09:23:56 -0800 Subject: [PATCH 2/4] Give every character the bucket of its Unicode Script property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit script_of gave the older UnicodeScript buckets (Latin through Modi) the Unicode blocks of their scripts, and the Script property only to the buckets added in 0.22.0. A letter outside those blocks was Other, so a buffer whose first letter sat there was guessed as Other and shaped under DFLT alone: Vietnamese in Latin Extended Additional lost its ligatures and kerning (Roboto "ỷ fi", the Windows 11 sans "Ựa"), Arabic Extended-A shaped left to right with no joining in all 7 fonts tried, and the Mongolian Supplement, the CJK extensions past B, Devanagari Extended and the other supplements lost their scripts. Every code point now takes the bucket of its Script property (Scripts.txt of Unicode 18.0.0, already the source of the table): Hiragana and Katakana fold into Han as before, and Common, Inherited, Unknown and the scripts without a bucket are Other. Script runs and the guessed script skip Unknown characters as well as Common and Inherited ones, as HarfBuzz's hb_buffer_guess_segment_properties does, and a kana run takes the kana tag by its Script property. The ordinal indicators U+00AA and U+00BA are Latin letters now; the hand-listed combining mark blocks still extend the run before them. The implicit vertical layout of Mongolian (no direction set, which HarfBuzz does not have) now applies to text that starts with a Mongolian character of the Mongolian block. It read the first character with a block bucket before, so with the Script property a leading space or a Mongolian Supplement ornament would have turned text vertical that was horizontal; text that starts with another character (a quotation mark, the Mongolian comma) now stays horizontal, as in HarfBuzz. Against HarfBuzz 14.5.0, texts whose first character sits outside the old blocks (11,043 rows over 30 fonts) go from 1,258 mismatches to 8, all Coptic letters, which have no bucket. --- src/buffer.rs | 60 ++++---- src/ot/mongolian.rs | 6 +- src/ot/tibetan.rs | 8 +- src/shape.rs | 4 +- src/shape/pipeline.rs | 33 ++-- src/shape/segment.rs | 15 +- src/unicode/mod.rs | 250 ++++++++++-------------------- src/unicode/script_table.rs | 56 +++---- src/unicode/script_tags.rs | 12 +- src/unicode/tests.rs | 95 +++++++++++- tests/mongolian_parity.rs | 38 ++++- tests/script_property_parity.rs | 262 ++++++++++++++++++++++++++++++++ tests/tibetan_parity.rs | 4 +- tests/unicode_table_gen.rs | 78 +++++----- 14 files changed, 598 insertions(+), 323 deletions(-) create mode 100644 tests/script_property_parity.rs diff --git a/src/buffer.rs b/src/buffer.rs index d0ea2a80..fa850be5 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -198,7 +198,7 @@ pub struct Buffer { /// `true` once a caller picked the direction with /// [`Buffer::set_direction`]. While `false`, `direction` is only /// the LTR default and `shape()` may choose vertical layout for - /// Mongolian-dominant text. [`Buffer::clear`] resets it. + /// Mongolian text. [`Buffer::clear`] resets it. pub(crate) direction_explicit: bool, /// Script the whole buffer shapes as, set by /// [`Buffer::set_script`]. `None` segments the text into script @@ -277,8 +277,8 @@ impl Buffer { /// [`Self::direction`] goes back to the [`Direction::Ltr`] default /// and [`Self::has_explicit_direction`] to `false`, so /// [`crate::shape`] once again picks the layout itself (vertical for - /// Mongolian-dominant text). The text, script, language, and - /// context are kept. + /// text that starts with a Mongolian character). The text, script, + /// language, and context are kept. /// /// ``` /// use sigilbuzz::{Buffer, Direction}; @@ -300,9 +300,11 @@ impl Buffer { /// [`Self::set_direction`], false while /// [`Self::direction`] only reports the LTR default. /// - /// [`crate::shape`] lays out Mongolian-dominant text vertically - /// (top to bottom) only while no direction is explicit; an - /// explicit [`Direction::Ltr`] keeps it horizontal. + /// [`crate::shape`] lays out text that starts with a character of + /// the Mongolian script from the Mongolian block (U+1800..U+18AF) + /// vertically (top to bottom) only while no direction is explicit; + /// an explicit [`Direction::Ltr`] keeps it horizontal. HarfBuzz has + /// no such default. /// /// ``` /// use sigilbuzz::{Buffer, Direction}; @@ -339,18 +341,21 @@ impl Buffer { } /// Splits the buffer's text into maximal script runs and yields - /// one `ScriptRun` per run. Consecutive codepoints sharing the - /// same resolved script collapse into a single run; `COMMON` - /// (digits, punctuation, ASCII space, ZWJ/ZWNJ/bidi marks, the - /// tatweel, the dandas) and `INHERITED` (combining marks) + /// one `ScriptRun` per run. Each character takes the bucket of its + /// Unicode Script property ([`crate::script_of`]), and consecutive + /// codepoints sharing the same resolved script collapse into a + /// single run; `COMMON` (digits, punctuation, ASCII space, + /// ZWJ/ZWNJ/bidi marks, the tatweel, the dandas), `INHERITED` + /// (combining marks), and `UNKNOWN` (unassigned and private-use) /// codepoints, as the Unicode Script property gives them, extend /// whichever real script ran before them, matching HarfBuzz's /// `select_shaper_for_script` segmentation. A Hangul tone mark /// (U+302E, U+302F) extends the run before it too. /// - /// A leading `COMMON`/`INHERITED` span before the first real - /// script codepoint joins that script's run, the way HarfBuzz - /// gives a buffer the script of its first non-`COMMON` character. + /// A leading `COMMON`/`INHERITED`/`UNKNOWN` span before the first + /// real script codepoint joins that script's run, the way HarfBuzz + /// gives a buffer the script of its first character of a real + /// script (`hb_buffer_guess_segment_properties`). /// Text with no real script at all is one `Script::Other` run /// with the default `DFLT` priority, same treatment HarfBuzz gives /// a pure-digits or pure-punctuation run. @@ -372,17 +377,11 @@ impl Buffer { let mut current: Option<(Script, usize)> = None; for (byte, ch) in self.text.char_indices() { let raw = script_of(ch); - // `COMMON` / `INHERITED` extend the previous real-script - // run if one exists. In sigilbuzz the only explicit bucket - // we keep for these characters is `Script::Other` (ASCII - // digits / punctuation land in `Script::Latin`; combining - // marks inherit their cluster base's script via the - // `script_of` range table). So the only codepoints we - // still need to actively extend are the Unicode format - // characters (ZWJ/ZWNJ/LRM/RLM/ALM) plus any other char - // that `script_of` could not classify. Everything with a - // real script bucket attaches normally through the - // script-equality test below. + // `COMMON` / `INHERITED` / `UNKNOWN` extend the previous + // real-script run if one exists (their `script_of` bucket + // is `Script::Other`). Everything with a real script + // bucket attaches through the script-equality test + // below. let resolved = if is_common_or_inherited(ch) { current.map_or(leading, |(s, _)| s) } else if is_hangul_tone_mark(ch) { @@ -517,9 +516,8 @@ pub fn script_priority_for(script: Script) -> &'static [[u8; 4]] { Script::Cyrillic => CYRL_PRIORITY, Script::Greek => GREK_PRIORITY, Script::Han => HANI_PRIORITY, - // The buckets that take their code points from the Unicode - // Script property try their own tag. Scripts sigilbuzz has no - // bucket for fall back to DFLT. + // The buckets added in 0.22.0 try their own tag. Scripts + // sigilbuzz has no bucket for fall back to DFLT. other => other.table_script_priority().unwrap_or(DFLT_ONLY), } } @@ -699,12 +697,8 @@ mod tests { #[test] fn script_runs_common_digits_stick_to_preceding_script() { - // "Price: ₪100 שלום": digits land in Script::Latin bucket - // via script_of, so they extend the Latin prefix. The shekel - // sign U+20AA falls outside our range table (Script::Other) - // but still extends the previous run because it is a COMMON - // codepoint in Unicode; sigilbuzz groups it with Latin here - // because `Script::Other` matches nothing-scripted neighbors. + // "Price: 100 שלום": the colon, the spaces, and the digits + // are COMMON, so they extend the Latin prefix. let mut b = Buffer::new(); b.push_str("Price: 100 \u{05E9}\u{05DC}\u{05D5}\u{05DD}"); let runs = b.script_runs(); diff --git a/src/ot/mongolian.rs b/src/ot/mongolian.rs index c65fbf96..78eab187 100644 --- a/src/ot/mongolian.rs +++ b/src/ot/mongolian.rs @@ -28,9 +28,9 @@ //! initial" behavior. //! - **Vertical default**. Mongolian is written top-to-bottom by //! default. While the caller has not chosen a direction, the -//! dispatcher in [`crate::shape`] lays a dominantly Mongolian run -//! out top to bottom: vertical metrics and the `vert` GSUB -//! feature. Consumers who want horizontal Mongolian set a +//! dispatcher in [`crate::shape`] lays text that starts with a +//! Mongolian character out top to bottom: vertical metrics and the +//! `vert` GSUB feature. Consumers who want horizontal Mongolian set a //! direction explicitly with //! [`crate::buffer::Buffer::set_direction`]: LTR keeps logical //! order, RTL returns the run reversed like any RTL run. diff --git a/src/ot/tibetan.rs b/src/ot/tibetan.rs index e98ef8ba..cc2968a8 100644 --- a/src/ot/tibetan.rs +++ b/src/ot/tibetan.rs @@ -29,10 +29,10 @@ use crate::tables::Gsub; /// for fonts that only carry features in the default LangSys. pub const TIBT_SCRIPT_PRIORITY: &[[u8; 4]] = &[*b"tibt", *b"DFLT"]; -/// True for every codepoint that is part of the Tibetan block. -/// Mirrors [`crate::unicode::script_of`]'s Tibetan arm so the -/// dispatcher can use it as a fast pre-filter without going through -/// the full script classifier. +/// True for every codepoint that is part of the Tibetan block +/// (U+0F00..U+0FFF), a fast block check. [`crate::unicode::script_of`] +/// reads the Script property instead, which gives the Common symbols +/// of the block (U+0FD5..U+0FD8) no Tibetan bucket. #[must_use] pub const fn is_tibetan(ch: char) -> bool { matches!(ch as u32, 0x0F00..=0x0FFF) diff --git a/src/shape.rs b/src/shape.rs index de0cf32a..544105e8 100644 --- a/src/shape.rs +++ b/src/shape.rs @@ -81,8 +81,8 @@ //! //! When the caller never set a direction //! ([`crate::Buffer::has_explicit_direction`] is false) the buffer -//! shapes as LTR, except that a Mongolian-dominant run switches to -//! vertical top-to-bottom layout. An explicit +//! shapes as LTR, except that text that starts with a Mongolian +//! character switches to vertical top-to-bottom layout. An explicit //! [`crate::Direction::Ltr`] keeps Mongolian horizontal. //! //! # Clusters and buffer flags diff --git a/src/shape/pipeline.rs b/src/shape/pipeline.rs index dc27a3e1..ae65723b 100644 --- a/src/shape/pipeline.rs +++ b/src/shape/pipeline.rs @@ -25,6 +25,17 @@ use crate::ot::arabic::{assign_from_types_in_context, JoiningContext, JoiningFor use crate::unicode::joining::{joining_type, JoiningType}; use crate::unicode::{script_of, Script}; +/// Whether text whose first character is `first` takes the implicit +/// vertical layout of Mongolian: `first` is a character of the +/// Mongolian script in the Mongolian block (U+1800..U+18AF: a letter, a +/// digit, the birga, a variation selector). The Common punctuation of +/// the block and the ornaments of the Mongolian Supplement start no +/// vertical text, and neither does a space, a digit, a quotation mark, +/// or a letter of another script before the Mongolian. +fn starts_mongolian(first: char) -> bool { + matches!(first as u32, 0x1800..=0x18AF) && script_of(first) == Script::Mongolian +} + /// Shapes `buffer` against `font` with optional feature overrides. /// /// Feature tags with `value: 0` disable the corresponding feature @@ -49,21 +60,13 @@ use crate::unicode::{script_of, Script}; // visible in one place. pub fn shape(font: &Font<'_>, buffer: &Buffer, features: &[Feature]) -> Result { // Vertical layout: explicit when the buffer direction is TTB/BTT, - // *implicit* when the run is dominantly Mongolian and the caller - // never chose a direction. Mongolian's traditional writing axis is - // top-to-bottom; auto-vertical here lets simple callers shape - // Mongolian without having to know the default. An explicit - // direction always wins, so `set_direction(Direction::Ltr)` gives - // horizontal Mongolian. - let mongolian_dominant = buffer - .text() - .chars() - .any(|c| crate::unicode::script_of(c) == crate::unicode::Script::Mongolian) - && buffer - .text() - .chars() - .find(|c| !matches!(crate::unicode::script_of(*c), crate::unicode::Script::Other)) - .is_some_and(|c| crate::unicode::script_of(c) == crate::unicode::Script::Mongolian); + // *implicit* for Mongolian text when the caller never chose a + // direction (see `starts_mongolian`). Mongolian's traditional + // writing axis is top-to-bottom; auto-vertical here lets simple + // callers shape Mongolian without having to know the default. + // HarfBuzz has no such default. An explicit direction always wins, + // so `set_direction(Direction::Ltr)` gives horizontal Mongolian. + let mongolian_dominant = buffer.text().chars().next().is_some_and(starts_mongolian); // The direction every later pass works with: the buffer's, or TTB // for implicit vertical Mongolian. let direction = if !buffer.has_explicit_direction() && mongolian_dominant { diff --git a/src/shape/segment.rs b/src/shape/segment.rs index 4de0b64b..a7d13725 100644 --- a/src/shape/segment.rs +++ b/src/shape/segment.rs @@ -32,14 +32,15 @@ pub(super) struct ProcessedSegment { /// The script tags a segment of `script` tries. The Han bucket also /// holds Hiragana and Katakana, which HarfBuzz tags `kana`, not `hani` /// (`hb_ot_tags_from_script`); a segment whose first script-bearing -/// character is kana takes `kana`, as HarfBuzz's buffer would. +/// character's Script property is Hiragana or Katakana takes `kana`, +/// as HarfBuzz's buffer would. fn segment_priority(script: Script, cps: &[char]) -> &'static [[u8; 4]] { const KANA_PRIORITY: &[[u8; 4]] = &[*b"kana", *b"DFLT"]; let kana = script == Script::Han && cps .iter() .find(|&&c| !is_common_for_segmentation(c)) - .is_some_and(|&c| matches!(c as u32, 0x3040..=0x30FF)); + .is_some_and(|&c| matches!(&crate::unicode::script_code(c), b"Hira" | b"Kana")); if kana { KANA_PRIORITY } else { @@ -49,8 +50,10 @@ fn segment_priority(script: Script, cps: &[char]) -> &'static [[u8; 4]] { /// Splits the post-cmap codepoint stream into [`Segment`]s whose /// scripts agree with the buffer-level [`crate::buffer::Buffer::script_runs`] -/// segmentation: COMMON codepoints (ASCII space/digits/punctuation, -/// ZWJ/ZWNJ/bidi marks) extend whichever real-script segment ran +/// segmentation: each codepoint takes the bucket of its Unicode Script +/// property, and COMMON, INHERITED, and UNKNOWN codepoints (ASCII +/// space/digits/punctuation, ZWJ/ZWNJ/bidi marks, combining marks, +/// private-use characters) extend whichever real-script segment ran /// before them, and so does a Hangul tone mark. A leading COMMON-only /// run joins the first real script after it, the way HarfBuzz gives a /// buffer the script of its first non-COMMON character, and text with @@ -109,8 +112,8 @@ pub(super) fn build_segments(codepoints: &[char]) -> Vec { } /// The script HarfBuzz's `hb_buffer_guess_segment_properties` gives a -/// buffer of `chars`: that of the first character that is not Common -/// or Inherited, or `None` when every character is. +/// buffer of `chars`: that of the first character that is not Common, +/// Inherited, or Unknown, or `None` when every character is. pub(crate) fn guess_script(chars: impl IntoIterator) -> Option