From 69d3a4e825c242f1378cd7d12ed9c68b38120725 Mon Sep 17 00:00:00 2001 From: WestWaters Date: Fri, 11 Sep 2026 01:08:48 -0700 Subject: [PATCH 1/3] unicode : add the K2-Horizon pre-tokenizer splitter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The K2-Horizon regex had no arm in unicode_regex_split_custom and fell through to the general std::regex fallback, which fails two ways. On MSVC std::regex rejects \p{...}, so no K2-Horizon GGUF loads on Windows at all: llama-quantize, llama-imatrix and llama-perplexity all abort with regex_error(error_escape) before a token is produced. Where the fallback does compile it is still wrong. unicode_regex_split collapses each codepoint to a single byte naming its Unicode category before matching, and U+200C/U+200D are category Control, which has no entry in k_ucat_cpt, so both become the 0xD0 fallback byte. The literal ‌ and ‍ alternatives in K2's regex can then never match and every ZWNJ or ZWJ ends a letter run. The splitter is the existing llama3 one with a single rule widened, since K2's regex differs from llama3's only in that a letter run also takes marks, ZWNJ and ZWJ. tests/test-unicode.cpp gains a case for this: it fails before the change with [Amy] [ZWNJ khaham] and passes after with the run intact. --- src/unicode.cpp | 148 +++++++++++++++++++++++++++++++++++++++++ tests/test-unicode.cpp | 30 +++++++-- 2 files changed, 171 insertions(+), 7 deletions(-) diff --git a/src/unicode.cpp b/src/unicode.cpp index 93996f9dd542..d2f166208f4a 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -470,6 +470,150 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & return bpe_offsets; } +static std::vector unicode_regex_split_custom_k2_horizon(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + // K2-Horizon: letter runs are (?:\p{L}|\p{M}|\u200C|\u200D)+ + auto _is_k2_letter = [&] (const size_t pos) -> bool { + const uint32_t c = _get_cpt(pos); + if (c == 0x200C || c == 0x200D) { + return true; + } + const auto f = _get_flags(pos); + return f.is_letter || f.is_accent_mark; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += _add_token(pos+2); + continue; + } + if (pos+2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += _add_token(pos+3); + continue; + } + } + } + + // regex: [^\r\n\p{L}\p{N}]?(?:\p{L}|\p{M}|\u200C|\u200D)+ + if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) { + if (_is_k2_letter(pos) || _is_k2_letter(pos+1)) { // one or more letters/marks/ZWNJ/ZWJ + pos++; + while (_is_k2_letter(pos)) { + pos++; + } + _add_token(pos); + continue; + } + } + + // regex: \p{N}{1,3} + if (flags.is_number) { + size_t ini = pos; + while (_get_flags(pos).is_number) { + if (++pos - ini >= 3 ) { + _add_token(pos); + ini = pos; + } + } + _add_token(pos); + continue; + } + + // regex: ?[^\s\p{L}\p{N}]+[\r\n]* + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + uint32_t cpt2 = _get_cpt(pos); + while (cpt2 == '\r' || cpt2 == '\n') { + cpt2 = _get_cpt(++pos); + } + _add_token(pos); + continue; + } + + size_t num_whitespaces = 0; + size_t last_end_r_or_n = 0; + while (_get_flags(pos+num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos+num_whitespaces); + if (cpt2 == '\r' || cpt2 == '\n') { + last_end_r_or_n = pos + num_whitespaces + 1; + } + num_whitespaces++; + } + + // regex: \s*[\r\n]+ + if (last_end_r_or_n > 0) { + pos = last_end_r_or_n; + _add_token(pos); + continue; + } + + // regex: \s+(?!\S) + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // regex: \s+ + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // no matches + _add_token(++pos); + } + } + + return bpe_offsets; +} + // Qwen2 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" static std::vector unicode_regex_split_custom_qwen2(const std::string & text, const std::vector & offsets) { std::vector bpe_offsets; // store the offset of each word @@ -1062,6 +1206,10 @@ static std::vector unicode_regex_split_custom(const std::string & text, } else if ( regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { bpe_offsets = unicode_regex_split_custom_qwen35(text, offsets); + } else if (regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + // K2-Horizon: llama3 splitter with marks + ZWNJ/ZWJ inside letter runs + // (the generic std::regex fallback cannot parse \p{..} on MSVC) + bpe_offsets = unicode_regex_split_custom_k2_horizon(text, offsets); } else if (regex_expr == "\\p{Han}+") { // K2's first pattern - handle all K2 patterns together bpe_offsets = unicode_regex_split_custom_kimi_k2(text, offsets); diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp index 2347d9000a8e..52edf759bbdf 100644 --- a/tests/test-unicode.cpp +++ b/tests/test-unicode.cpp @@ -4,15 +4,13 @@ #include #include -int main() { - const std::vector regex_exprs = { - "[~][A-Za-z]+| ?[\\p{S}]+|\\s+", - }; - const std::vector expected = { " ~", "foo" }; - const auto actual = unicode_regex_split(" ~foo", regex_exprs, false); +static int check(const char * name, const std::string & text, + const std::vector & regex_exprs, + const std::vector & expected) { + const auto actual = unicode_regex_split(text, regex_exprs, false); if (actual != expected) { - fprintf(stderr, "unexpected split:"); + fprintf(stderr, "%s: unexpected split:", name); for (const auto & piece : actual) { fprintf(stderr, " [%s]", piece.c_str()); } @@ -22,3 +20,21 @@ int main() { return 0; } + +int main() { + int n_fail = 0; + + n_fail += check("simple", " ~foo", + { "[~][A-Za-z]+| ?[\\p{S}]+|\\s+" }, + { " ~", "foo" }); + + // K2-Horizon letter runs take marks and ZWNJ/ZWJ, so a ZWNJ must not end a run. + // "A" + "mi" + U+200C + "khaham" + " " + "1" + const std::string k2_text = "Aمی‌خواهم 1"; + + n_fail += check("k2-horizon zwnj", k2_text, + { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, + { "Aمی‌خواهم", " ", "1" }); + + return n_fail; +} From a8104b5532af306b72e2e811d46e23fce9531514 Mon Sep 17 00:00:00 2001 From: "Natani L. Mayday" <71436458+TaskPuppyNatani@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:18:00 -0700 Subject: [PATCH 2/3] tests: expand K2 Horizon unicode splitter coverage --- tests/test-unicode.cpp | 216 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 192 insertions(+), 24 deletions(-) diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp index 52edf759bbdf..3ea2991fb46a 100644 --- a/tests/test-unicode.cpp +++ b/tests/test-unicode.cpp @@ -1,40 +1,208 @@ #include "../src/unicode.h" #include +#include #include #include -static int check(const char * name, const std::string & text, - const std::vector & regex_exprs, - const std::vector & expected) { - const auto actual = unicode_regex_split(text, regex_exprs, false); +int main() { + { + const std::vector regex_exprs = { + "[~][A-Za-z]+| ?[\\p{S}]+|\\s+", + }; + const std::vector expected = { " ~", "foo" }; + const auto actual = unicode_regex_split(" ~foo", regex_exprs, false); - if (actual != expected) { - fprintf(stderr, "%s: unexpected split:", name); - for (const auto & piece : actual) { - fprintf(stderr, " [%s]", piece.c_str()); + if (actual != expected) { + fprintf(stderr, "unexpected split:"); + for (const auto & piece : actual) { + fprintf(stderr, " [%s]", piece.c_str()); + } + fprintf(stderr, "\n"); + return 1; } - fprintf(stderr, "\n"); - return 1; } - return 0; -} + { + const std::vector regex_exprs = { + "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|" + "[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|" + "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|" + "\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; -int main() { - int n_fail = 0; + const std::string input = "ab\u200ccd ef\u200dgh cafe\u0301"; + const std::vector expected = { + "ab\u200ccd", + " ef\u200dgh", + " cafe\u0301", + }; + + try { + const auto actual = unicode_regex_split(input, regex_exprs, false); + + if (actual != expected) { + fprintf(stderr, "unexpected K2-Horizon split:"); + for (const auto & piece : actual) { + fprintf(stderr, " [%s]", piece.c_str()); + } + fprintf(stderr, "\n"); + return 1; + } + } catch (const std::exception & e) { + fprintf(stderr, "K2-Horizon regex split threw exception: %s\n", e.what()); + return 1; + } + } + + + { + const std::vector llama3_regex = { + "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|" + "[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|" + "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|" + "\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + + const std::vector k2_horizon_regex = { + "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|" + "[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|" + "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|" + "\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + + const std::vector inputs = { + "Hello, world!", + "can't won't we're they'll", + "123 1234 1234567", + "Hello!!!\\nNext line", + "alpha beta gamma", + " café résumé", + }; + + for (const auto & input : inputs) { + const auto llama3 = unicode_regex_split(input, llama3_regex, false); + const auto k2 = unicode_regex_split(input, k2_horizon_regex, false); + + if (llama3 != k2) { + fprintf(stderr, "K2-Horizon diverged from Llama 3 for ordinary input: %s\n", input.c_str()); + + fprintf(stderr, "Llama 3:"); + for (const auto & piece : llama3) { + fprintf(stderr, " [%s]", piece.c_str()); + } + + fprintf(stderr, "\nK2-Horizon:"); + for (const auto & piece : k2) { + fprintf(stderr, " [%s]", piece.c_str()); + } + + fprintf(stderr, "\n"); + return 1; + } + } + } + + + { + const std::vector k2_horizon_regex = { + "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|" + "[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|" + "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|" + "\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + + auto check_k2 = [&] ( + const char * name, + const std::string & input, + const std::vector & expected) { + const auto actual = unicode_regex_split(input, k2_horizon_regex, false); - n_fail += check("simple", " ~foo", - { "[~][A-Za-z]+| ?[\\p{S}]+|\\s+" }, - { " ~", "foo" }); + if (actual != expected) { + fprintf(stderr, "K2-Horizon reference mismatch for %s\n", name); - // K2-Horizon letter runs take marks and ZWNJ/ZWJ, so a ZWNJ must not end a run. - // "A" + "mi" + U+200C + "khaham" + " " + "1" - const std::string k2_text = "Aمی‌خواهم 1"; + fprintf(stderr, "expected:"); + for (const auto & piece : expected) { + fprintf(stderr, " [%s]", piece.c_str()); + } - n_fail += check("k2-horizon zwnj", k2_text, - { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, - { "Aمی‌خواهم", " ", "1" }); + fprintf(stderr, "\nactual:"); + for (const auto & piece : actual) { + fprintf(stderr, " [%s]", piece.c_str()); + } - return n_fail; + fprintf(stderr, "\n"); + return false; + } + + return true; + }; + + if (!check_k2( + "ZWNJ", + "ab\u200ccd", + { "ab\u200ccd" })) { + return 1; + } + + if (!check_k2( + "ZWJ", + "ef\u200dgh", + { "ef\u200dgh" })) { + return 1; + } + + if (!check_k2( + "combined ZWNJ/ZWJ", + "ab\u200ccd ef\u200dgh", + { "ab\u200ccd", " ef\u200dgh" })) { + return 1; + } + + // The reference tokenizer NFC-normalizes cafe + combining acute + // to the composed form before pre-tokenization. + if (!check_k2( + "NFC accent", + "caf\u00e9", + { "caf\u00e9" })) { + return 1; + } + + if (!check_k2( + "Persian ZWNJ", + "\u0645\u06cc\u200c\u0631\u0648\u0645", + { "\u0645\u06cc\u200c\u0631\u0648\u0645" })) { + return 1; + } + + if (!check_k2( + "Devanagari ZWJ", + "\u0915\u094d\u200d\u0937", + { "\u0915\u094d\u200d\u0937" })) { + return 1; + } + + if (!check_k2( + "contractions", + "can't won't we're they'll", + { "can", "'t", " won", "'t", " we", "'re", " they", "'ll" })) { + return 1; + } + + if (!check_k2( + "numbers", + "123 1234 1234567", + { "123", " ", "123", "4", " ", "123", "456", "7" })) { + return 1; + } + + if (!check_k2( + "punctuation/newline", + "Hello!!!\nNext line", + { "Hello", "!!!\n", "Next", " line" })) { + return 1; + } + } + + return 0; } From e78bd9435e3773974f3d080b635dcf17c52e7c00 Mon Sep 17 00:00:00 2001 From: aaryamonvikram Date: Thu, 17 Sep 2026 17:06:37 +0400 Subject: [PATCH 3/3] unicode: handle K2 Horizon case folding and empty input Assisted-by: Codex --- src/llama-vocab.cpp | 2 +- src/unicode.cpp | 11 ++++++++++- tests/test-unicode.cpp | 16 +++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 4d054db403fc..38e5623374c2 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -537,7 +537,7 @@ struct llm_tokenizer_bpe : llm_tokenizer { break; case LLAMA_VOCAB_PRE_TYPE_K2_HORIZON: regex_exprs = { - "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }; break; case LLAMA_VOCAB_PRE_TYPE_WHITESPACE: diff --git a/src/unicode.cpp b/src/unicode.cpp index d2f166208f4a..10e8b408fbdd 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -520,6 +520,9 @@ static std::vector unicode_regex_split_custom_k2_horizon(const std::stri // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive if (cpt == '\'' && pos+1 < offset_end) { uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); + if (cpt_next == 0x017F) { + cpt_next = 's'; // Unicode case-folding of long s + } if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { pos += _add_token(pos+2); continue; @@ -1206,7 +1209,9 @@ static std::vector unicode_regex_split_custom(const std::string & text, } else if ( regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { bpe_offsets = unicode_regex_split_custom_qwen35(text, offsets); - } else if (regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + } else if ( + regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" || + regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { // K2-Horizon: llama3 splitter with marks + ZWNJ/ZWJ inside letter runs // (the generic std::regex fallback cannot parse \p{..} on MSVC) bpe_offsets = unicode_regex_split_custom_k2_horizon(text, offsets); @@ -1362,6 +1367,10 @@ bool unicode_cpt_is_han(uint32_t cpt) { } std::vector unicode_regex_split(const std::string & text, const std::vector & regex_exprs, bool byte_encode) { + if (text.empty()) { + return {}; + } + // unicode categories static const std::map k_ucat_enum = { { "\\p{N}", unicode_cpt_flags::NUMBER }, diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp index 3ea2991fb46a..72a0f6ed0e09 100644 --- a/tests/test-unicode.cpp +++ b/tests/test-unicode.cpp @@ -106,7 +106,7 @@ int main() { { const std::vector k2_horizon_regex = { - "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|" + "(?i:'s|'t|'re|'ve|'m|'ll|'d)|" "[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|" "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|" "\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", @@ -189,6 +189,20 @@ int main() { return 1; } + if (!check_k2( + "Unicode contraction", + "'\u017fa", + { "'\u017f", "a" })) { + return 1; + } + + if (!check_k2( + "empty input", + "", + { })) { + return 1; + } + if (!check_k2( "numbers", "123 1234 1234567",