From 1a0390e8e59febaa11bb0be59601bae098efb595 Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Sat, 22 Aug 2026 22:58:45 +0200 Subject: [PATCH 1/7] Add malformed policy --- API.md | 491 +++++++++--- CONTRIBUTING.md | 5 +- README.md | 20 +- ROADMAP.md | 11 +- examples/example.c | 11 +- examples/python/example.py | 13 +- examples/rust/example.rs | 13 + examples/zig/example.zig | 6 + fuzz/fuzz.c | 96 ++- src/api/index.ts | 68 +- src/api/mojibake.d.ts | 24 +- src/api/tests/index.ts | 49 ++ src/bidi.c | 36 +- src/break-sentence.c | 84 +- src/break-word.c | 103 ++- src/case.c | 108 ++- src/caseless.c | 30 +- src/collation.c | 64 +- src/cpp/mojibake.hpp | 234 ++++-- src/encoding.c | 728 +++++++++++++++--- src/filter.c | 90 ++- src/idna.c | 26 +- src/locales.c | 3 +- src/mojibake-internal.h | 4 + src/mojibake.c | 4 +- src/mojibake.h | 81 +- src/next.c | 37 +- src/normalization.c | 158 +++- src/quick-check.c | 64 +- src/security.c | 26 +- src/segmentation.c | 88 ++- src/shell/commands/break.c | 5 +- src/shell/commands/character.c | 2 +- src/shell/commands/emoji.c | 8 +- src/shell/commands/filter.c | 4 +- src/shell/commands/normalize.c | 16 +- src/shell/commands/string.c | 4 +- src/shell/shell.c | 14 +- src/string.c | 35 +- src/terminal-width.c | 16 +- src/ui/Mojibake/Mojibake/CaseView.swift | 4 +- .../Mojibake/Mojibake/CharacterDetails.swift | 4 +- src/ui/Mojibake/Mojibake/CollationView.swift | 4 +- .../Mojibake/EncodingInspectorView.swift | 8 +- src/ui/Mojibake/Mojibake/FilterView.swift | 4 +- .../Mojibake/Mojibake/MojibakeSupport.swift | 8 +- .../Mojibake/Mojibake/NormalizationView.swift | 8 +- .../Mojibake/Mojibake/TerminalWidthView.swift | 4 +- src/utf.h | 123 +-- tests/bidi.c | 5 + tests/break-sentence.c | 45 +- tests/break-word.c | 58 +- tests/case.c | 73 +- tests/collation.c | 167 ++-- tests/embedded-null.c | 23 +- tests/encoding.c | 271 ++++++- tests/example.c | 109 ++- tests/ext/cpp/normalization.cpp | 55 ++ tests/filter.c | 259 ++++--- tests/mojibake.c | 60 +- tests/next.c | 40 +- tests/normalization.c | 188 +++-- tests/quick-check.c | 5 + tests/segmentation.c | 67 +- tests/string.c | 14 +- tests/terminal-width.c | 140 ++-- tests/utf.c | 21 +- tests/utils/utils.c | 10 +- utils/generate/functions.ts | 468 ++++++++--- 69 files changed, 3779 insertions(+), 1345 deletions(-) diff --git a/API.md b/API.md index 0e544bb2..27e6cc6e 100644 --- a/API.md +++ b/API.md @@ -75,7 +75,8 @@ the best approach if you don't want to change it. So this is ok: ```c // See, the output is UTF-16. -mjb_filter("Hello", 5, MJB_ENC_UTF_8, MJB_FILTER_SPACES, MJB_ENC_UTF_16LE, &result); +mjb_filter("Hello", 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_FILTER_SPACES, + MJB_ENC_UTF_16LE, &result, NULL); ``` ### Every string passed is simply a stream of bytes @@ -97,8 +98,10 @@ equivalent: const char *hello = "Hello"; mjb_encoding enc = MJB_ENC_UTF_8; -mjb_normalize(hello, 5, enc, MJB_NORMALIZATION_NFC, enc, &result); -mjb_normalize(hello, MJB_NUL_TERMINATED, enc, MJB_NORMALIZATION_NFC, enc, &result); +mjb_normalize(hello, 5, enc, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, enc, &result, + NULL); +mjb_normalize(hello, MJB_NUL_TERMINATED, enc, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, enc, &result, NULL); const char hello_16[] = { 0x48, 0x00, 0x65, 0x00, 0x6C, 0x00, @@ -109,8 +112,10 @@ const char hello_16[] = { enc = MJB_ENC_UTF_16LE; // sizeof(uint16_t) is 2 -mjb_normalize(hello_16, 5 * sizeof(uint16_t), enc, MJB_NORMALIZATION_NFC, enc, &result); -mjb_normalize(hello_16, MJB_NUL_TERMINATED, enc, MJB_NORMALIZATION_NFC, enc, &result); +mjb_normalize(hello_16, 5 * sizeof(uint16_t), enc, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, enc, &result, NULL); +mjb_normalize(hello_16, MJB_NUL_TERMINATED, enc, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, enc, &result, NULL); ``` ### Status return @@ -139,9 +144,11 @@ The functions return a `_mjb_status_` and accept these arguments: 1. The input string 2. The _length_ of the input string (`byte_length`) 3. The encoding of the input string (`encoding`) -4. The needed _arguments_ of the function, if any -5. The encoding you want to be used for the output string -6. A `mjb_result` pointer to store the result +4. A malformed-input policy when the operation can recover malformed text +5. The needed _arguments_ of the function, if any +6. The encoding you want to be used for the output string +7. A `mjb_result` pointer to store the result +8. An optional `mjb_diagnostic` pointer for the first malformed input sequence See for example the [`mjb_normalize`](#mjb_normalize), [`mjb_filter`](#mjb_filter) functions. @@ -173,15 +180,18 @@ Example of the [`mjb_normalize`](#mjb_normalize) function. ```c mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_encoding encoding, -mjb_normalization form, mjb_encoding output_encoding, mjb_result *result); +mjb_malformed_policy malformed_policy, mjb_normalization form, mjb_encoding output_encoding, +mjb_result *result, mjb_diagnostic *diagnostic); ``` 1. `buffer`: a block of memory, `uint8_t` (ASCII, UTF-8), `uint16_t` (UTF-16), `uint32_t` (UTF-32) 2. `byte_length`: the length in _bytes_ of `buffer` -3. `form`: the normalization -4. `encoding`: the encoding of `buffer` -5. `output_encoding`: the encoding of _output_ you want. -6. `results`: a pointer to a struct the function will fill +3. `encoding`: the encoding of `buffer` +4. `malformed_policy`: whether malformed input stops, is replaced, or is skipped +5. `form`: the normalization +6. `output_encoding`: the encoding of _output_ you want +7. `result`: a pointer to a struct the function will fill +8. `diagnostic`: an optional pointer that receives the first malformed input sequence If you want to normalize the UTF-8 encoded `Cafe\xCC\x81` string to `NFC`, this is what you need to do: @@ -191,8 +201,8 @@ const char *input = "Cafe\xCC\x81"; // UTF-8 string of: "Cafe" + U+0301 COMBININ mjb_result result; -if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -210,8 +220,8 @@ you need to `mjb_result_free` it. This way the output buffer will be encoded in UTF-16LE. ```c -if(mjb_normalize(input, strlen(input), MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_16LE, - &result) != MJB_STATUS_OK) { +if(mjb_normalize(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK) { return 1; } ``` @@ -229,11 +239,16 @@ To count the Unicode codepoints, use `mjb_codepoint_count`. ```c size_t count = 0; -mjb_codepoint_count("H\xC3\xA9ll\xC3\xB6", 7, MJB_ENC_UTF_8, &count); // 5 characters -mjb_codepoint_count("H\0\xE9\0l\0l\0\xF6\0", 10, MJB_ENC_UTF_16LE, &count); // 5 characters -mjb_codepoint_count("\0H\0\xE9\0l\0l\0\xF6", 10, MJB_ENC_UTF_16BE, &count); // 5 characters -mjb_codepoint_count("H\0\0\0\xE9\0\0\0l\0\0\0l\0\0\0\xF6\0\0\0", 20, MJB_ENC_UTF_32LE, &count); // 5 characters -mjb_codepoint_count("\0\0\0H\0\0\0\xE9\0\0\0l\0\0\0l\0\0\0\xF6", 20, MJB_ENC_UTF_32BE, &count); // 5 characters +mjb_codepoint_count("H\xC3\xA9ll\xC3\xB6", 7, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL); // 5 characters +mjb_codepoint_count("H\0\xE9\0l\0l\0\xF6\0", 10, MJB_ENC_UTF_16LE, + MJB_MALFORMED_STOP, &count, NULL); // 5 characters +mjb_codepoint_count("\0H\0\xE9\0l\0l\0\xF6", 10, MJB_ENC_UTF_16BE, + MJB_MALFORMED_STOP, &count, NULL); // 5 characters +mjb_codepoint_count("H\0\0\0\xE9\0\0\0l\0\0\0l\0\0\0\xF6\0\0\0", 20, + MJB_ENC_UTF_32LE, MJB_MALFORMED_STOP, &count, NULL); // 5 characters +mjb_codepoint_count("\0\0\0H\0\0\0\xE9\0\0\0l\0\0\0l\0\0\0\xF6", 20, + MJB_ENC_UTF_32BE, MJB_MALFORMED_STOP, &count, NULL); // 5 characters ``` Functions that handle a string, as `mjb_normalize`, `mjb_convert_encoding` has always a `_into(...)` @@ -242,8 +257,10 @@ alternative that do not allocate the results for you, but you provide the buffer Example for the function: ```c -mjb_status mjb_convert_encoding_into(const char *buffer, size_t byte_length, mjb_encoding encoding, -mjb_encoding output_encoding, void *output, size_t *output_size); +mjb_status mjb_convert_encoding_into(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_encoding output_encoding, void *output, size_t *output_size, + mjb_diagnostic *diagnostic); ``` These are the rules, all the functions are equal: @@ -266,7 +283,7 @@ size_t required = 0; // Get the length. if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, NULL, &required) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, NULL, &required, NULL) != MJB_STATUS_OK) { return 1; } @@ -274,12 +291,12 @@ char *output = (char *)malloc(required); size_t capacity = required; if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, output, &capacity) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, output, &capacity, NULL) != MJB_STATUS_OK) { return 1; } // UTF-16LE payload bytes (no terminator): 8 -printf("UTF-16LE payload bytes (no terminator): %zu", output_size) +printf("UTF-16LE payload bytes (no terminator): %zu", capacity); free(output); ``` @@ -455,26 +472,32 @@ mjb_status mjb_normalize( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_normalization form, mjb_encoding output_encoding, - mjb_result *result + mjb_result *result, + mjb_diagnostic *diagnostic ); ``` -Normalize a string to the requested Unicode normalization form. If the input is already normalized and no encoding conversion is needed, the input buffer is returned as-is in `result->output` with `result->transformed` set to false, without allocating. +Normalize a string to the requested Unicode normalization form. If the input is already normalized and no encoding conversion is needed, the input buffer is returned as-is in `result->output` with `result->transformed` set to false, without allocating. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. - `buffer` - The string to normalize - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `form` - The normalization form to use - `output_encoding` - The output encoding of the string - `result` - The pointer to store the result. If `result->transformed` is true, `result->output` is library-allocated and must be freed with `mjb_result_free(result)` +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The string was normalized (or already normal) -- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, the buffer is invalid, or the malformed policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information - `MJB_STATUS_INVALID_FORM` - `form` is not NFC, NFD, NFKC, or NFKD +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_OVERFLOW` - The output size would overflow - `MJB_STATUS_NO_MEMORY` - Allocation failed @@ -484,8 +507,8 @@ Normalize a string to the requested Unicode normalization form. If the input is const char *input = "Cafe\xCC\x81"; // "Cafe" + U+0301 COMBINING ACUTE ACCENT mjb_result result; -if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -508,10 +531,12 @@ mjb_status mjb_normalize_into( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_normalization form, mjb_encoding output_encoding, void *output, - size_t *output_size + size_t *output_size, + mjb_diagnostic *diagnostic ); ``` @@ -520,18 +545,20 @@ Normalize a string using the same Unicode normalization forms and encoding rules - `buffer` - The string to normalize - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `form` - The normalization form to use - `output_encoding` - The output encoding of the string - `output` - The caller-provided output buffer, or NULL to query the required size. The caller retains ownership - `output_size` - The input capacity and output required or written byte count +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The required size was returned or the normalized string was written -- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, the buffer is invalid, or the malformed policy is invalid - `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information - `MJB_STATUS_INVALID_FORM` - `form` is not NFC, NFD, NFKC, or NFKD -- `MJB_STATUS_MALFORMED_INPUT` - The input contains an ill-formed code-unit sequence +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_UNSUPPORTED` - The requested output encoding cannot represent a normalized codepoint - `MJB_STATUS_OVERFLOW` - The required output size would overflow - `MJB_STATUS_NO_MEMORY` - Temporary composition allocation failed @@ -543,15 +570,16 @@ Normalize a string using the same Unicode normalization forms and encoding rules const char *input = "Cafe\xCC\x81"; // "Cafe" + U+0301 COMBINING ACUTE ACCENT size_t output_size = 0; -if(mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } char output[5]; if(output_size > sizeof(output) || mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, output, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } @@ -572,20 +600,34 @@ mjb_status mjb_filter( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_filter_flags filters, mjb_encoding output_encoding, - mjb_result *result + mjb_result *result, + mjb_diagnostic *diagnostic ); ``` -`MJB_FILTER_LIMIT_COMBINING` removes combining marks after the first `MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in an emitted run. This is useful for reducing Zalgo-style text while keeping ordinary accents and stacked marks. +`MJB_FILTER_LIMIT_COMBINING` removes combining marks after the first `MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in an emitted run. This is useful for reducing Zalgo-style text while keeping ordinary accents and stacked marks. Malformed subsequences are stopped, replaced, or skipped according to `malformed_policy`; `diagnostic` records the first one encountered. - `buffer` - The string to filter - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `filters` - The filters to use - `output_encoding` - The output encoding of the string - `result` - The pointer to store the result. If `result->transformed` is true, `result->output` is library-allocated and must be freed with `mjb_result_free(result)` +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL + +**Returns** + +- `MJB_STATUS_OK` - The filtered string was returned +- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, the buffer is invalid, or the malformed policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` +- `MJB_STATUS_UNSUPPORTED` - The requested output encoding cannot represent a filtered codepoint +- `MJB_STATUS_OVERFLOW` - The output size would overflow +- `MJB_STATUS_NO_MEMORY` - Allocation failed **Example** @@ -594,7 +636,8 @@ const char *mixed_whitespace = "Hello\t\t\n\nworld"; mjb_result result; if(mjb_filter(mixed_whitespace, strlen(mixed_whitespace), MJB_ENC_UTF_8, - MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, &result) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, &result, + NULL) != MJB_STATUS_OK) { return 1; } @@ -605,8 +648,8 @@ mjb_result_free(&result); const char *controls = "\x1\x2\t\n\v\f\r\x1f"; -if(mjb_filter(controls, strlen(controls), MJB_ENC_UTF_8, MJB_FILTER_CONTROLS, - MJB_ENC_UTF_8, &result) != MJB_STATUS_OK) { +if(mjb_filter(controls, strlen(controls), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_CONTROLS, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -627,28 +670,33 @@ mjb_status mjb_filter_into( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_filter_flags filters, mjb_encoding output_encoding, void *output, - size_t *output_size + size_t *output_size, + mjb_diagnostic *diagnostic ); ``` -Apply the same filters as `mjb_filter` without allocating the final output buffer. Set `output` to NULL to query the required size. If `output` is non-NULL, `*output_size` supplies its capacity; on return it contains the required size when the buffer is too small, or the written size on success. Terminators are excluded from the byte count and are not written. No bytes are written when capacity is insufficient. Filtering itself does not allocate, but `MJB_FILTER_NORMALIZE` may allocate temporary normalization storage. `MJB_FILTER_LIMIT_COMBINING` keeps the first `MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in each emitted run. +Apply the same filters as `mjb_filter` without allocating the final output buffer. Set `output` to NULL to query the required size. If `output` is non-NULL, `*output_size` supplies its capacity; on return it contains the required size when the buffer is too small, or the written size on success. Terminators are excluded from the byte count and are not written. No bytes are written when capacity is insufficient. Filtering itself does not allocate, but `MJB_FILTER_NORMALIZE` may allocate temporary normalization storage. `MJB_FILTER_LIMIT_COMBINING` keeps the first `MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in each emitted run. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. - `buffer` - The string to filter - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `filters` - The filters to use - `output_encoding` - The output encoding of the string - `output` - The caller-provided output buffer, or NULL to query the required size. The caller retains ownership - `output_size` - The input capacity and output required or written byte count +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The required size was returned or the filtered string was written -- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, the buffer is invalid, or the malformed policy is invalid - `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_UNSUPPORTED` - The requested output encoding cannot represent a filtered codepoint - `MJB_STATUS_OVERFLOW` - The required output size would overflow - `MJB_STATUS_NO_MEMORY` - Temporary normalization allocation failed @@ -660,15 +708,17 @@ Apply the same filters as `mjb_filter` without allocating the final output buffe const char *input = "Hello\t\t\nworld"; size_t output_size = 0; -if(mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, MJB_FILTER_COLLAPSE_SPACES, - MJB_ENC_UTF_8, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, NULL, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } char output[11]; if(output_size > sizeof(output) || mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, output, + &output_size, NULL) != MJB_STATUS_OK) { return 1; } @@ -687,23 +737,29 @@ mjb_status mjb_nfkc_casefold( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, - mjb_result *result + mjb_result *result, + mjb_diagnostic *diagnostic ); ``` -Apply the normative `NFKC_Casefold` mapping and normalize the result to NFC. This transform performs compatibility folding, full default case folding, and removal of default-ignorable codepoints. It is intended for identifier comparison and is not locale-sensitive. +Apply the normative `NFKC_Casefold` mapping and normalize the result to NFC. This transform performs compatibility folding, full default case folding, and removal of default-ignorable codepoints. It is intended for identifier comparison and is not locale-sensitive. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. - `buffer` - The string to transform - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `output_encoding` - The output encoding of the string - `result` - The pointer to store the result. If `result->transformed` is true, `result->output` is library-allocated and must be freed with `mjb_result_free(result)` +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The transformed string was returned -- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, the buffer is invalid, or the malformed policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_OVERFLOW` - The output size would overflow - `MJB_STATUS_UNSUPPORTED` - The transform did not stabilize - `MJB_STATUS_NO_MEMORY` - Allocation failed @@ -714,8 +770,8 @@ Apply the normative `NFKC_Casefold` mapping and normalize the result to NFC. Thi const char *input = "Stra\xC3\x9F" "e\xC2\xAD"; mjb_result result; -if(mjb_nfkc_casefold(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_nfkc_casefold(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -737,9 +793,11 @@ mjb_status mjb_nfkc_casefold_into( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, void *output, - size_t *output_size + size_t *output_size, + mjb_diagnostic *diagnostic ); ``` @@ -748,14 +806,18 @@ Apply the same normative `NFKC_Casefold` transform as `mjb_nfkc_casefold`. Set ` - `buffer` - The string to transform - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `output_encoding` - The output encoding of the string - `output` - The caller-provided output buffer, or NULL to query the required size. The caller retains ownership - `output_size` - The input capacity and output required or written byte count +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The required size was returned or the transformed string was written -- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, the buffer is invalid, or the malformed policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_OVERFLOW` - The required output size would overflow - `MJB_STATUS_UNSUPPORTED` - The transform did not stabilize - `MJB_STATUS_NO_MEMORY` - Temporary allocation failed @@ -767,15 +829,15 @@ Apply the same normative `NFKC_Casefold` transform as `mjb_nfkc_casefold`. Set ` const char *input = "Stra\xC3\x9F" "e\xC2\xAD"; size_t output_size = 0; -if(mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, - NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } char output[7]; if(output_size > sizeof(output) || mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, output, &output_size, NULL) != MJB_STATUS_OK) { return 1; } @@ -1128,6 +1190,119 @@ const char utf16be[] = "\xFE\xFF\0H\0i"; // BOM + "Hi" in UTF-16BE printf("UTF-16: %s", mjb_is_utf16(utf16be, sizeof(utf16be) - 1) ? "yes" : "no"); ``` +## `mjb_string_validate` + +Validate a complete Unicode code-unit sequence. + +```c +mjb_status mjb_string_validate( + const char *buffer, + size_t byte_length, + mjb_encoding encoding, + mjb_diagnostic *diagnostic +); +``` + +Validate the complete input without producing output. Empty input is well-formed. On malformed input, `diagnostic` identifies the first maximal ill-formed subsequence. Generic UTF-16 and UTF-32 require a byte-order mark. + +- `buffer` - The string to validate +- `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit +- `encoding` - The encoding of the string +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL + +**Returns** + +- `MJB_STATUS_OK` - The complete input is well-formed +- `MJB_STATUS_INVALID_ARGUMENT` - `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ENCODING` - The encoding is unsupported, or generic UTF-16/UTF-32 has no byte-order mark +- `MJB_STATUS_MALFORMED_INPUT` - The first malformed subsequence is described by `diagnostic` + +**Example** + +```c +const char invalid[] = "\xE2\x82"; +mjb_diagnostic diagnostic; + +if(mjb_string_validate(invalid, sizeof(invalid) - 1, MJB_ENC_UTF_8, + &diagnostic) != MJB_STATUS_MALFORMED_INPUT || diagnostic.byte_offset != 0) { + return 1; +} +``` + +See also: [`mjb_decode_next`](#mjb_decode_next), [`mjb_decode_previous`](#mjb_decode_previous), [`mjb_is_utf8`](#mjb_is_utf8). + +## `mjb_decode_next` + +Decode the next codepoint from a string. + +```c +mjb_status mjb_decode_next( + const char *buffer, + size_t byte_length, + mjb_encoding encoding, + mjb_malformed_policy malformed_policy, + size_t *offset, + mjb_codepoint *codepoint, + mjb_diagnostic *diagnostic +); +``` + +Decode one codepoint and advance `offset`. With `MJB_MALFORMED_STOP`, malformed input returns `MJB_STATUS_MALFORMED_INPUT`; `offset` still advances over the malformed subsequence so decoding can resume. `MJB_MALFORMED_REPLACE` returns U+FFFD, while `MJB_MALFORMED_SKIP` advances until a valid codepoint or end of input. A diagnostic is reported for replacement and skipping even when the function returns success. + +- `buffer` - The string to decode +- `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit +- `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled +- `offset` - The input byte offset and the byte offset following the decoded subsequence +- `codepoint` - Where to store the decoded codepoint +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL + +**Returns** + +- `MJB_STATUS_OK` - A codepoint was decoded +- `MJB_STATUS_END_OF_INPUT` - No codepoint remains +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` +- `MJB_STATUS_INVALID_ARGUMENT` - An argument or policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - The encoding cannot be decoded + +See also: [`mjb_decode_previous`](#mjb_decode_previous), [`mjb_string_validate`](#mjb_string_validate), [`mjb_codepoint_count`](#mjb_codepoint_count). + +## `mjb_decode_previous` + +Decode the previous codepoint from a string. + +```c +mjb_status mjb_decode_previous( + const char *buffer, + size_t byte_length, + mjb_encoding encoding, + mjb_malformed_policy malformed_policy, + size_t *offset, + mjb_codepoint *codepoint, + mjb_diagnostic *diagnostic +); +``` + +Decode backward from `offset`, using the same malformed-input policies and diagnostic contract as `mjb_decode_next`. On success, `offset` is the first byte of the decoded codepoint. Start with the input byte length to iterate from the end. + +- `buffer` - The string to decode +- `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit +- `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled +- `offset` - The input byte offset and the start of the decoded subsequence +- `codepoint` - Where to store the decoded codepoint +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL + +**Returns** + +- `MJB_STATUS_OK` - A codepoint was decoded +- `MJB_STATUS_END_OF_INPUT` - No codepoint precedes `offset` +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` +- `MJB_STATUS_INVALID_ARGUMENT` - An argument or policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - The encoding cannot be decoded + +See also: [`mjb_decode_next`](#mjb_decode_next), [`mjb_string_validate`](#mjb_string_validate), [`mjb_codepoint_count`](#mjb_codepoint_count). + ## `mjb_codepoint_count` Count the codepoints in a string. @@ -1137,22 +1312,27 @@ mjb_status mjb_codepoint_count( const char *buffer, size_t byte_length, mjb_encoding encoding, - size_t *count + mjb_malformed_policy malformed_policy, + size_t *count, + mjb_diagnostic *diagnostic ); ``` -Count the number of Unicode codepoints in a string. Malformed code-unit sequences count per the library replacement policy, and an incomplete trailing sequence does not add a codepoint. On failure, `count` is set to zero. +Count the number of decoded Unicode codepoints in a string. Malformed subsequences are stopped, replaced, or skipped according to `malformed_policy`. A replacement counts as one codepoint. On failure, `count` is set to zero. - `buffer` - The string to count - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `count` - The number of codepoints to store; set to zero on failure +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The count was computed - `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, or `buffer` is NULL with a non-zero size - `MJB_STATUS_INVALID_ENCODING` - The encoding is not a supported input encoding +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` **Example** @@ -1163,14 +1343,16 @@ const char *utf8 = "H\xC3\xA9ll\xC3\xB6"; // 7 bytes const char utf16le[] = "H\0\xE9\0l\0l\0\xF6\0"; // 10 bytes size_t count; -if(mjb_codepoint_count(utf8, 7, MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_codepoint_count(utf8, 7, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } // 5 UTF-8 characters printf("%zu UTF-8 characters", count); -if(mjb_codepoint_count(utf16le, 10, MJB_ENC_UTF_16LE, &count) != MJB_STATUS_OK) { +if(mjb_codepoint_count(utf16le, 10, MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -1189,19 +1371,34 @@ mjb_status mjb_for_each_codepoint( const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_for_each_codepoint_fn callback + mjb_malformed_policy malformed_policy, + mjb_for_each_codepoint_fn callback, + mjb_diagnostic *diagnostic ); ``` +Decode the string according to `malformed_policy` and call the callback for every resulting codepoint. The first malformed subsequence is reported in `diagnostic` even when it is replaced or skipped. + - `buffer` - The string to check - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `callback` - The function to call for each codepoint +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL + +**Returns** + +- `MJB_STATUS_OK` - Every decoded codepoint was visited +- `MJB_STATUS_INVALID_ARGUMENT` - The buffer, callback, or malformed policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - The encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` +- `MJB_STATUS_CALLBACK_STOPPED` - The callback returned false **Example** ```c -mjb_status status = mjb_for_each_codepoint("ABC", 3, MJB_ENC_UTF_8, NULL); +mjb_status status = mjb_for_each_codepoint("ABC", 3, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, NULL, NULL); // A callback is required: yes bool callback_required = status == MJB_STATUS_INVALID_ARGUMENT; @@ -1455,24 +1652,29 @@ mjb_status mjb_convert_encoding( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, - mjb_result *result + mjb_result *result, + mjb_diagnostic *diagnostic ); ``` -Convert a string between the supported encodings (UTF-8, UTF-16LE/BE, UTF-32LE/BE). Generic UTF-16/UTF-32 input consumes a leading BOM as the encoding scheme signature and uses it to resolve byte order. Explicit-endian input preserves an initial U+FEFF as text. Generic UTF-16/UTF-32 without a BOM, and generic UTF-16/UTF-32 output, are rejected because the byte order is not specified. +Convert a string between the supported encodings (UTF-8, UTF-16LE/BE, UTF-32LE/BE). Generic UTF-16/UTF-32 input consumes a leading BOM as the encoding scheme signature and uses it to resolve byte order. Explicit-endian input preserves an initial U+FEFF as text. Generic UTF-16/UTF-32 without a BOM, and generic UTF-16/UTF-32 output, are rejected because the byte order is not specified. Malformed source subsequences are stopped, replaced, or skipped according to `malformed_policy`. - `buffer` - The string to convert - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The input encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `output_encoding` - The output encoding of the string - `result` - The pointer to store the result. If `result->transformed` is true, `result->output` is library-allocated and must be freed with `mjb_result_free(result)` +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The string was converted -- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, `buffer` is NULL with a non-zero size, or the input is not valid in the source encoding -- `MJB_STATUS_INVALID_ENCODING` - A generic UTF-16/UTF-32 encoding did not provide enough byte order information +- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, `buffer` is NULL with a non-zero size, or the malformed policy is invalid +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_UNSUPPORTED` - The requested encoding conversion is not supported - `MJB_STATUS_OVERFLOW` - The output size would overflow - `MJB_STATUS_NO_MEMORY` - Allocation failed @@ -1483,8 +1685,8 @@ Convert a string between the supported encodings (UTF-8, UTF-16LE/BE, UTF-32LE/B const char *input = "caf\xC3\xA9"; mjb_result result; -if(mjb_convert_encoding(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, &result) != MJB_STATUS_OK) { +if(mjb_convert_encoding(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -1504,9 +1706,11 @@ mjb_status mjb_convert_encoding_into( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, void *output, - size_t *output_size + size_t *output_size, + mjb_diagnostic *diagnostic ); ``` @@ -1515,15 +1719,18 @@ Convert a string using the same encoding and BOM rules as `mjb_convert_encoding` - `buffer` - The string to convert - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The input encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `output_encoding` - The output encoding of the string - `output` - The caller-provided output buffer, or NULL to query the required size. The caller retains ownership - `output_size` - The input capacity and output required or written byte count +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The required size was returned or the string was converted - `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, or `buffer` is NULL with a non-zero size -- `MJB_STATUS_INVALID_ENCODING` - A generic UTF-16/UTF-32 encoding did not provide enough byte order information +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_UNSUPPORTED` - The requested encoding conversion is not supported - `MJB_STATUS_OVERFLOW` - The required output size would overflow - `MJB_STATUS_OUTPUT_TOO_SMALL` - The output capacity is smaller than the required byte count @@ -1534,15 +1741,16 @@ Convert a string using the same encoding and BOM rules as `mjb_convert_encoding` const char *input = "caf\xC3\xA9"; size_t output_size = 0; -if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } unsigned char output[8]; if(output_size > sizeof(output) || mjb_convert_encoding_into(input, strlen(input), - MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, output, &output_size) != MJB_STATUS_OK) { + MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, output, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } @@ -1678,27 +1886,31 @@ mjb_status mjb_collation_key( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, - mjb_result *result + mjb_result *result, + mjb_diagnostic *diagnostic ); ``` -Generate a binary sort key for a string. Sort keys of different strings can be compared with `memcmp` and yield the same order as `mjb_collation_compare` when both use the same variable weighting and strength. Useful when the same strings are compared many times, such as sorting or database indexing. Empty input and non-empty input with no effective weights at the selected strength both produce a zero-length key. If `MJB_FEATURE_COLLATION=0` the function always returns `MJB_STATUS_FEATURE_NOT_ENABLED`. +Generate a binary sort key for a string. Sort keys of different strings can be compared with `memcmp` and yield the same order as `mjb_collation_compare` when both use the same variable weighting and strength. Useful when the same strings are compared many times, such as sorting or database indexing. Empty input and non-empty input with no effective weights at the selected strength both produce a zero-length key. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. If `MJB_FEATURE_COLLATION=0` the function always returns `MJB_STATUS_FEATURE_NOT_ENABLED`. - `buffer` - The string to generate the sort key for - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `variable_weighting` - The variable weighting strategy - `strength` - The maximum collation level to include - `result` - The pointer to store the binary sort key. If `result->transformed` is true, `result->output` is library-allocated and must be freed with `mjb_result_free(result)` +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The sort key was generated -- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, `buffer` is NULL with a non-zero size, or an option is invalid +- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, the buffer is invalid, the malformed policy is invalid, or a collation option is invalid - `MJB_STATUS_INVALID_ENCODING` - The input encoding is invalid or lacks byte-order information -- `MJB_STATUS_MALFORMED_INPUT` - The input contains an ill-formed code-unit sequence +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_OVERFLOW` - The sort key size would overflow - `MJB_STATUS_NO_MEMORY` - Allocation failed - `MJB_STATUS_FEATURE_NOT_ENABLED` - The library was compiled with `MJB_FEATURE_COLLATION=0` @@ -1709,7 +1921,8 @@ Generate a binary sort key for a string. Sort keys of different strings can be c mjb_result key; if(mjb_collation_key("r\xC3\xA9sum\xC3\xA9", 8, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &key) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &key, + NULL) != MJB_STATUS_OK) { return 1; } @@ -1731,10 +1944,12 @@ mjb_status mjb_collation_key_into( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, void *output, - size_t *output_size + size_t *output_size, + mjb_diagnostic *diagnostic ); ``` @@ -1743,17 +1958,19 @@ Generate the same binary sort key as `mjb_collation_key` without allocating the - `buffer` - The string to generate the sort key for - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `variable_weighting` - The variable weighting strategy - `strength` - The maximum collation level to include - `output` - The caller-provided binary output buffer, or NULL to query its size. The caller retains ownership - `output_size` - The input capacity and output required or written byte count +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The required size was returned or the binary sort key was written -- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, `buffer` is NULL with a non-zero size, or an option is invalid +- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, the buffer is invalid, the malformed policy is invalid, or a collation option is invalid - `MJB_STATUS_INVALID_ENCODING` - The input encoding is invalid -- `MJB_STATUS_MALFORMED_INPUT` - The input contains an ill-formed code-unit sequence +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_OVERFLOW` - The required key size would overflow - `MJB_STATUS_NO_MEMORY` - Temporary allocation failed - `MJB_STATUS_OUTPUT_TOO_SMALL` - The output capacity is smaller than the required byte count @@ -1765,16 +1982,18 @@ Generate the same binary sort key as `mjb_collation_key` without allocating the const char *input = "r\xC3\xA9sum\xC3\xA9"; size_t output_size = 0; -if(mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } unsigned char output[64]; if(output_size > sizeof(output) || mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, output, - &output_size) != MJB_STATUS_OK) { + &output_size, NULL) != MJB_STATUS_OK) { return 1; } @@ -1795,25 +2014,31 @@ mjb_status mjb_map_case( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_map_case_type type, mjb_encoding output_encoding, - mjb_result *result + mjb_result *result, + mjb_diagnostic *diagnostic ); ``` -Convert a string to uppercase, lowercase, titlecase, or its case-folded form. Full case mappings are applied, including special casing and conditional mappings, so the output may have a different length than the input. Titlecase uses UAX #29 word boundaries: the first cased character in each word segment is titlecased, and subsequent characters in that segment are lowercased. Casing is tailored by the process-global locale set with `mjb_set_locale`: the default `MJB_LOCALE_EN` uses default non-Turkic mappings. `MJB_LOCALE_TR` and `MJB_LOCALE_AZ` apply Turkish/Azerbaijani dotted-I casing and Turkic `T` case-folding mappings. `MJB_LOCALE_LT` applies Lithuanian dot-above casing rules, while case folding remains the default non-Turkic mapping. +Convert a string to uppercase, lowercase, titlecase, or its case-folded form. Full case mappings are applied, including special casing and conditional mappings, so the output may have a different length than the input. Titlecase uses UAX #29 word boundaries: the first cased character in each word segment is titlecased, and subsequent characters in that segment are lowercased. Casing is tailored by the process-global locale set with `mjb_set_locale`: the default `MJB_LOCALE_EN` uses default non-Turkic mappings. `MJB_LOCALE_TR` and `MJB_LOCALE_AZ` apply Turkish/Azerbaijani dotted-I casing and Turkic `T` case-folding mappings. `MJB_LOCALE_LT` applies Lithuanian dot-above casing rules, while case folding remains the default non-Turkic mapping. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. - `buffer` - The string to change case - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `type` - The type of case change - `output_encoding` - The output encoding of the string - `result` - The pointer to store the result. If `result->transformed` is true, `result->output` is library-allocated and must be freed with `mjb_result_free(result)` +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The case conversion succeeded -- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, `buffer` is NULL with a non-zero size, or `type` is not a valid case type +- `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, the buffer is invalid, the malformed policy is invalid, or `type` is not a valid case type +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_NO_MEMORY` - Allocation failed **Example** @@ -1822,8 +2047,8 @@ Convert a string to uppercase, lowercase, titlecase, or its case-folded form. Fu const char *input = "Stra\xC3\x9F""e"; // "Straße" mjb_result result; -if(mjb_map_case(input, strlen(input), MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_map_case(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_CASE_UPPER, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -1846,10 +2071,12 @@ mjb_status mjb_map_case_into( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_map_case_type type, mjb_encoding output_encoding, void *output, - size_t *output_size + size_t *output_size, + mjb_diagnostic *diagnostic ); ``` @@ -1858,15 +2085,19 @@ Apply the same full, special, conditional, titlecase, locale-sensitive, and case - `buffer` - The string to change case - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `type` - The type of case change - `output_encoding` - The output encoding of the string - `output` - The caller-provided output buffer, or NULL to query the required size. The caller retains ownership - `output_size` - The input capacity and output required or written byte count +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The required size was returned or the case-mapped string was written -- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, `buffer` is NULL with a non-zero size, or `type` is invalid +- `MJB_STATUS_INVALID_ARGUMENT` - `output_size` is NULL, the buffer is invalid, the malformed policy is invalid, or `type` is invalid +- `MJB_STATUS_INVALID_ENCODING` - An encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` - `MJB_STATUS_UNSUPPORTED` - The requested output encoding cannot represent a mapped codepoint - `MJB_STATUS_OVERFLOW` - The required output size would overflow - `MJB_STATUS_OUTPUT_TOO_SMALL` - The output capacity is smaller than the required byte count @@ -1877,15 +2108,16 @@ Apply the same full, special, conditional, titlecase, locale-sensitive, and case const char *input = "Stra\xC3\x9F""e"; // "Straße" size_t output_size = 0; -if(mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_CASE_UPPER, MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } char output[7]; if(output_size > sizeof(output) || mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_CASE_UPPER, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_8, output, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } @@ -2248,22 +2480,27 @@ mjb_status mjb_sentence_count( const char *buffer, size_t byte_length, mjb_encoding encoding, - size_t *count + mjb_malformed_policy malformed_policy, + size_t *count, + mjb_diagnostic *diagnostic ); ``` -Count the sentence segments produced by the default Unicode sentence-boundary rules. The default rules carry no abbreviation list, so text such as `Dr. Smith` counts as two sentences. Malformed code-unit sequences are segmented per the library replacement policy. On failure, `count` is set to zero. +Count the sentence segments produced by the default Unicode sentence-boundary rules. The default rules carry no abbreviation list, so text such as `Dr. Smith` counts as two sentences. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. On failure, `count` is set to zero. - `buffer` - The string to count - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `count` - The number of sentence segments to store; set to zero on failure +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The count was computed -- `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, the buffer is invalid, or the malformed policy is invalid - `MJB_STATUS_INVALID_ENCODING` - The encoding is not a supported input encoding +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` **Example** @@ -2271,7 +2508,8 @@ Count the sentence segments produced by the default Unicode sentence-boundary ru const char *input = "Hello. How are you? Fine!"; size_t count; -if(mjb_sentence_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_sentence_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -2361,22 +2599,27 @@ mjb_status mjb_grapheme_count( const char *buffer, size_t byte_length, mjb_encoding encoding, - size_t *count + mjb_malformed_policy malformed_policy, + size_t *count, + mjb_diagnostic *diagnostic ); ``` -Count user-perceived characters: the number of extended grapheme cluster segments in the string. Malformed code-unit sequences are segmented per the library replacement policy, and an incomplete trailing sequence does not add a cluster. On failure, `count` is set to zero. +Count user-perceived characters: the number of extended grapheme cluster segments in the string. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. On failure, `count` is set to zero. - `buffer` - The string to count - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `count` - The number of grapheme clusters to store; set to zero on failure +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The count was computed -- `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, the buffer is invalid, or the malformed policy is invalid - `MJB_STATUS_INVALID_ENCODING` - The encoding is not a supported input encoding +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` **Example** @@ -2384,7 +2627,8 @@ Count user-perceived characters: the number of extended grapheme cluster segment const char *input = "A\xF0\x9F\x87\xAE\xF0\x9F\x87\xB9"; // A🇮🇹 size_t count; -if(mjb_grapheme_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_grapheme_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -2466,22 +2710,27 @@ mjb_status mjb_word_count( const char *buffer, size_t byte_length, mjb_encoding encoding, - size_t *count + mjb_malformed_policy malformed_policy, + size_t *count, + mjb_diagnostic *diagnostic ); ``` -Count the words in a string: the word-break segments that contain at least one alphabetic or numeric character. Punctuation, whitespace, and symbol segments are not counted, so `Hello, world!` counts as two words. Hyphenated compounds count each part, matching the default Unicode word-boundary rules. Unlike `mjb_truncate_word`, whose `max_segments` counts every raw segment, this function skips non-word segments. For scripts segmented by dictionary lookup in other implementations, such as Chinese, Japanese, Thai, Lao, Khmer, and Burmese, the count approximates one word per character: Mojibake does not use frequency dictionaries to keep the size of the library small. Malformed code-unit sequences are segmented per the library replacement policy. On failure, `count` is set to zero. +Count the words in a string: the word-break segments that contain at least one alphabetic or numeric character. Punctuation, whitespace, and symbol segments are not counted, so `Hello, world!` counts as two words. Hyphenated compounds count each part, matching the default Unicode word-boundary rules. Unlike `mjb_truncate_word`, whose `max_segments` counts every raw segment, this function skips non-word segments. For scripts segmented by dictionary lookup in other implementations, such as Chinese, Japanese, Thai, Lao, Khmer, and Burmese, the count approximates one word per character: Mojibake does not use frequency dictionaries to keep the size of the library small. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. On failure, `count` is set to zero. - `buffer` - The string to count - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `count` - The number of word-like segments to store; set to zero on failure +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The count was computed -- `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `count` is NULL, the buffer is invalid, or the malformed policy is invalid - `MJB_STATUS_INVALID_ENCODING` - The encoding is not a supported input encoding +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` **Example** @@ -2489,7 +2738,8 @@ Count the words in a string: the word-break segments that contain at least one a const char *input = "Hello, world! It works."; size_t count; -if(mjb_word_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_word_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -2560,6 +2810,8 @@ Resolve the embedding levels of a paragraph following the Unicode Bidirectional - `MJB_STATUS_OK` - The paragraph was resolved - `MJB_STATUS_INVALID_ARGUMENT` - `result` is NULL, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ENCODING` - The input encoding is invalid or lacks byte-order information +- `MJB_STATUS_MALFORMED_INPUT` - The input contains an ill-formed code-unit sequence - `MJB_STATUS_OVERFLOW` - The paragraph size would overflow - `MJB_STATUS_NO_MEMORY` - Allocation failed @@ -3582,28 +3834,32 @@ mjb_status mjb_terminal_width( const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, mjb_terminal_width_profile profile, - size_t *width + size_t *width, + mjb_diagnostic *diagnostic ); ``` -Estimate the number of fixed terminal cells occupied by printable, single-line text. The input is normalized to NFC so canonically equivalent text has the same width. Combining and format characters occupy no additional cells, listed emoji-presentation sequences occupy two cells, and text-presentation sequences retain their East Asian Width. This is a deterministic terminal policy, not a measurement of proportional glyph advances. Use grapheme boundaries for cursor movement, selection, deletion, and user-perceived character counts. Controls, line separators, and paragraph separators are rejected because their effect depends on terminal state. On failure, `width` is set to zero. +Estimate the number of fixed terminal cells occupied by printable, single-line text. The input is normalized to NFC so canonically equivalent text has the same width. Combining and format characters occupy no additional cells, listed emoji-presentation sequences occupy two cells, and text-presentation sequences retain their East Asian Width. This is a deterministic terminal policy, not a measurement of proportional glyph advances. Use grapheme boundaries for cursor movement, selection, deletion, and user-perceived character counts. Controls, line separators, and paragraph separators are rejected because their effect depends on terminal state. Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. On failure, `width` is set to zero. - `buffer` - The printable, single-line string to measure - `byte_length` - The length of the string in bytes, or `MJB_NUL_TERMINATED` to determine it from an encoding-aware NUL code unit - `encoding` - The encoding of the string +- `malformed_policy` - How malformed code-unit sequences are handled - `profile` - The terminal-width profile for ambiguous-width characters - `width` - The number of terminal cells to store; set to zero on failure +- `diagnostic` - Where to store the first malformed-input diagnostic, or NULL **Returns** - `MJB_STATUS_OK` - The width was computed -- `MJB_STATUS_INVALID_ARGUMENT` - `width` is NULL, the profile is invalid, or `buffer` is NULL with a non-zero size +- `MJB_STATUS_INVALID_ARGUMENT` - `width` is NULL, the profile or malformed policy is invalid, or the buffer is invalid - `MJB_STATUS_INVALID_ENCODING` - The encoding is invalid or lacks required byte-order information - `MJB_STATUS_UNSUPPORTED` - The input contains a control, line separator, or paragraph separator - `MJB_STATUS_NO_MEMORY` - NFC normalization could not allocate memory - `MJB_STATUS_OVERFLOW` - The width would overflow -- `MJB_STATUS_MALFORMED_INPUT` - The input contains a malformed code-unit sequence +- `MJB_STATUS_MALFORMED_INPUT` - Malformed input was encountered with `MJB_MALFORMED_STOP` **Example** @@ -3611,8 +3867,8 @@ Estimate the number of fixed terminal cells occupied by printable, single-line t const char *input = "A\xE7\x95\x8C"; // A界 size_t width; -if(mjb_terminal_width(input, strlen(input), MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &width) != MJB_STATUS_OK) { +if(mjb_terminal_width(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &width, NULL) != MJB_STATUS_OK) { return 1; } @@ -3777,8 +4033,9 @@ Free the memory allocated for a `mjb_result`. The `result` pointer is set to NUL ```c mjb_result result; -if(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, - &result) != MJB_STATUS_OK || mjb_result_free(&result) != MJB_STATUS_OK) { +if(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK || + mjb_result_free(&result) != MJB_STATUS_OK) { return 1; } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ac0d356b..0b70793f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,8 +192,9 @@ new assertion: ```c size_t count = 0; -ATT_ASSERT_STATUS(mjb_codepoint_count("Hello, test", 11, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, - "UTF-8 Hello, test status") +ATT_ASSERT_STATUS(mjb_codepoint_count("Hello, test", 11, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL), + MJB_STATUS_OK, "UTF-8 Hello, test status") ATT_ASSERT(count, (size_t)11, "UTF-8 Hello, test") ``` diff --git a/README.md b/README.md index 722597a9..03c0b6ee 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ int main(int argc, char *const argv[]) { mjb_result result; // Normalize example: in NFC e + ◌́ -> é (U+00E9) - if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { + if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -44,7 +44,8 @@ int main(int argc, char *const argv[]) { // Codepoint count example: mjb_codepoint_count counts Unicode codepoints, not bytes. size_t codepoint_count = 0; - if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, &codepoint_count) == MJB_STATUS_OK) { + if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &codepoint_count, NULL) == MJB_STATUS_OK) { printf("\"%s\" encoded in UTF-8 is %zu bytes long, and %zu codepoints long\n", mojibake, length, codepoint_count); } @@ -54,8 +55,8 @@ int main(int argc, char *const argv[]) { const char *case_input = "Straße"; // NFKC casefold example: in NFKC casefold, ß -> ss - if(mjb_nfkc_casefold(case_input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { + if(mjb_nfkc_casefold(case_input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -168,10 +169,11 @@ and header: `mojibake.c` and `mojibake.h`. Zero dependencies. **Integration** - **Encodings**: the API accepts and outputs UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE - strings, with encoding detection and conversion (`mjb_detect_encoding`, - `mjb_convert_encoding`, `mjb_convert_encoding_into`) -- **Parsing and string functions**: codepoint-by-codepoint iteration (`mjb_for_each_codepoint`) - and codepoint counting (`mjb_codepoint_count`) + strings, with detailed validation and explicit stop, replacement, or skip policies during + conversion (`mjb_string_validate`, `mjb_convert_encoding`, `mjb_convert_encoding_into`) +- **Parsing and string functions**: forward and reverse decoding with malformed-input diagnostics + (`mjb_decode_next`, `mjb_decode_previous`), codepoint callbacks (`mjb_for_each_codepoint`), and + policy-aware codepoint counting (`mjb_codepoint_count`) - **Locales**: strict BCP 47 language tag parsing (`mjb_locale_parse`) - **Embeddable**: context-aware custom allocators (`mjb_set_allocator`), build-time feature flags to trim table size, a C++17 wrapper (`src/cpp/mojibake.hpp`), a CLI tool (`src/shell`), and a diff --git a/ROADMAP.md b/ROADMAP.md index 68295970..1313d8f9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,20 +7,17 @@ This is the Mojibake roadmap. For great justice. 1. **Complete UTS #39 identifier checks** Generate compact tables from `IdentifierStatus.txt` and `IdentifierType.txt`, then add `mjb_identifier_check` and `mjb_string_restriction_level`. -2. **Detailed validation and decoder iteration** - Add `mjb_string_validate`, `mjb_decode_next`, and `mjb_decode_previous`. Use a shared diagnostic - result with the first failing byte (or code-unit offset?) and a precise malformed-input kind. -3. **Finish typed UCD access** +2. **Finish typed UCD access** Add typed getters for code-point, code-point-sequence, and string-valued properties. Follow with character age, bidi mirror, modern/alias/extended names, and reverse character-name lookup. Is this needed? -4. **Explicit locale operations** +3. **Explicit locale operations** Implement the currently unsupported `mjb_locale_canonicalize` using a pinned IANA Language Subtag Registry snapshot. -5. **Reusable configurable collators** +4. **Reusable configurable collators** Introduce immutable DUCET collator objects that encapsulate strength and variable weighting, then add case ordering, numeric collation, and normalization options. -6. **Streaming processing** +5. **Streaming processing** Add stateful `init`/`feed`/`finish` APIs for decoding and conversion, normalization, casing, and segmentation, using caller-kind of API. diff --git a/examples/example.c b/examples/example.c index e8333242..76561aca 100644 --- a/examples/example.c +++ b/examples/example.c @@ -20,8 +20,8 @@ int main(int argc, char *const argv[]) { mjb_result result; // Normalize example: in NFC e + ◌́ -> é (U+00E9) - if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result) != - MJB_STATUS_OK) { + if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -37,7 +37,8 @@ int main(int argc, char *const argv[]) { // Codepoint count example: mjb_codepoint_count counts Unicode codepoints, not bytes. size_t codepoint_count = 0; - if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, &codepoint_count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &codepoint_count, + NULL) != MJB_STATUS_OK) { return 1; } @@ -49,8 +50,8 @@ int main(int argc, char *const argv[]) { const char *case_input = "Straße"; // NFKC casefold example: in NFKC casefold, ß -> ss - if(mjb_nfkc_casefold(case_input, strlen(case_input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, &result) != - MJB_STATUS_OK) { + if(mjb_nfkc_casefold(case_input, strlen(case_input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } diff --git a/examples/python/example.py b/examples/python/example.py index ca9630d2..e5e150eb 100644 --- a/examples/python/example.py +++ b/examples/python/example.py @@ -16,6 +16,7 @@ MJB_STATUS_OK = 0 MJB_ENC_UTF_8 = 0x2 +MJB_MALFORMED_STOP = 0 MJB_NORMALIZATION_NFC = 0 @@ -46,7 +47,9 @@ def load_mojibake() -> ctypes.CDLL: ctypes.c_int, ctypes.c_int, ctypes.c_int, + ctypes.c_int, ctypes.POINTER(MjbResult), + ctypes.c_void_p, ] library.mjb_normalize.restype = ctypes.c_int @@ -56,7 +59,9 @@ def load_mojibake() -> ctypes.CDLL: ctypes.c_size_t, ctypes.c_int, ctypes.c_int, + ctypes.c_int, ctypes.POINTER(MjbResult), + ctypes.c_void_p, ] library.mjb_nfkc_casefold.restype = ctypes.c_int @@ -65,7 +70,9 @@ def load_mojibake() -> ctypes.CDLL: ctypes.c_char_p, ctypes.c_size_t, ctypes.c_int, + ctypes.c_int, ctypes.POINTER(ctypes.c_size_t), + ctypes.c_void_p, ] library.mjb_codepoint_count.restype = ctypes.c_int @@ -88,9 +95,11 @@ def normalize(library: ctypes.CDLL, input_bytes: bytes) -> bytes: input_bytes, len(input_bytes), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, ctypes.byref(result), + None, ) if status != MJB_STATUS_OK: @@ -105,8 +114,10 @@ def nfkc_casefold(library: ctypes.CDLL, input_bytes: bytes) -> bytes: input_bytes, len(input_bytes), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, ctypes.byref(result), + None, ) if status != MJB_STATUS_OK: @@ -143,7 +154,7 @@ def main() -> int: # Codepoint count example: mjb_codepoint_count counts Unicode codepoints, not bytes. codepoint_count = ctypes.c_size_t(0) status = library.mjb_codepoint_count( - mojibake, len(mojibake), MJB_ENC_UTF_8, ctypes.byref(codepoint_count) + mojibake, len(mojibake), MJB_ENC_UTF_8, 0, ctypes.byref(codepoint_count), None ) if status != MJB_STATUS_OK: diff --git a/examples/rust/example.rs b/examples/rust/example.rs index a8512c69..9d6ca124 100644 --- a/examples/rust/example.rs +++ b/examples/rust/example.rs @@ -9,6 +9,7 @@ use std::slice; const MJB_STATUS_OK: c_int = 0; const MJB_ENC_UTF_8: c_int = 0x2; const MJB_NORMALIZATION_NFC: c_int = 0; +const MJB_MALFORMED_STOP: c_int = 0; #[repr(C)] struct MjbResult { @@ -48,24 +49,30 @@ extern "C" { buffer: *const c_char, byte_length: usize, encoding: c_int, + malformed_policy: c_int, form: c_int, output_encoding: c_int, result: *mut MjbResult, + diagnostic: *mut std::ffi::c_void, ) -> c_int; fn mjb_nfkc_casefold( buffer: *const c_char, byte_length: usize, encoding: c_int, + malformed_policy: c_int, output_encoding: c_int, result: *mut MjbResult, + diagnostic: *mut std::ffi::c_void, ) -> c_int; fn mjb_codepoint_count( buffer: *const c_char, byte_length: usize, encoding: c_int, + malformed_policy: c_int, count: *mut usize, + diagnostic: *mut std::ffi::c_void, ) -> c_int; fn mjb_result_free(result: *mut MjbResult) -> c_int; @@ -92,9 +99,11 @@ fn run() -> bool { input.as_ptr().cast(), input.len(), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &mut normalized, + std::ptr::null_mut(), ) } != MJB_STATUS_OK { @@ -116,7 +125,9 @@ fn run() -> bool { mojibake.as_ptr().cast(), mojibake.len(), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &mut codepoint_count, + std::ptr::null_mut(), ) }; assert_eq!(status, MJB_STATUS_OK, "mjb_codepoint_count failed"); @@ -134,8 +145,10 @@ fn run() -> bool { case_input.as_ptr().cast(), case_input.len(), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &mut casefolded, + std::ptr::null_mut(), ) } != MJB_STATUS_OK { diff --git a/examples/zig/example.zig b/examples/zig/example.zig index 6855774f..7e8177c2 100644 --- a/examples/zig/example.zig +++ b/examples/zig/example.zig @@ -23,9 +23,11 @@ pub fn main(init: std.process.Init) !u8 { input.ptr, input.len, mjb.MJB_ENC_UTF_8, + mjb.MJB_MALFORMED_STOP, mjb.MJB_NORMALIZATION_NFC, mjb.MJB_ENC_UTF_8, &result, + null, ) != mjb.MJB_STATUS_OK) { return 1; } @@ -46,7 +48,9 @@ pub fn main(init: std.process.Init) !u8 { mojibake.ptr, mojibake.len, mjb.MJB_ENC_UTF_8, + mjb.MJB_MALFORMED_STOP, &codepoint_count, + null, ) != mjb.MJB_STATUS_OK) { return error.CodepointCountFailed; } @@ -66,8 +70,10 @@ pub fn main(init: std.process.Init) !u8 { case_input.ptr, case_input.len, mjb.MJB_ENC_UTF_8, + mjb.MJB_MALFORMED_STOP, mjb.MJB_ENC_UTF_8, &result, + null, ) != mjb.MJB_STATUS_OK) { return 1; } diff --git a/fuzz/fuzz.c b/fuzz/fuzz.c index cdfd0a76..004253ec 100644 --- a/fuzz/fuzz.c +++ b/fuzz/fuzz.c @@ -356,6 +356,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { static const mjb_encoding encodings[] = { MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, MJB_ENC_UTF_16BE, MJB_ENC_UTF_32LE, MJB_ENC_UTF_32BE, MJB_ENC_ASCII }; mjb_encoding encoding = encodings[variant % 6]; + mjb_malformed_policy malformed_policy = (mjb_malformed_policy)(variant % 3); // Exercise the language-sensitive casing and folding paths too. static const mjb_locale locales[] = { MJB_LOCALE_EN, MJB_LOCALE_TR, MJB_LOCALE_AZ, @@ -380,20 +381,22 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { switch(selector % 23) { case 0: { // Normalization, all four forms - if(mjb_normalize(buffer, size, encoding, (mjb_normalization)(variant % 4), - MJB_ENC_UTF_8, &result) == MJB_STATUS_OK) { + if(mjb_normalize(buffer, size, encoding, malformed_policy, + (mjb_normalization)(variant % 4), MJB_ENC_UTF_8, &result, + NULL) == MJB_STATUS_OK) { mjb_result_free(&result); } size_t required = 0; - if(mjb_normalize_into(buffer, size, encoding, (mjb_normalization)(variant % 4), - MJB_ENC_UTF_8, NULL, &required) == MJB_STATUS_OK && + if(mjb_normalize_into(buffer, size, encoding, malformed_policy, + (mjb_normalization)(variant % 4), MJB_ENC_UTF_8, NULL, &required, + NULL) == MJB_STATUS_OK && required <= 4096) { char output[4096]; size_t capacity = required; - fuzz_sink += (size_t)mjb_normalize_into(buffer, size, encoding, - (mjb_normalization)(variant % 4), MJB_ENC_UTF_8, output, &capacity); + fuzz_sink += (size_t)mjb_normalize_into(buffer, size, encoding, malformed_policy, + (mjb_normalization)(variant % 4), MJB_ENC_UTF_8, output, &capacity, NULL); } break; @@ -407,20 +410,22 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } case 2: { // Case conversion and folding, all transforming types - if(mjb_map_case(buffer, size, encoding, (mjb_map_case_type)(1 + (variant % 5)), - MJB_ENC_UTF_8, &result) == MJB_STATUS_OK) { + if(mjb_map_case(buffer, size, encoding, malformed_policy, + (mjb_map_case_type)(1 + (variant % 5)), MJB_ENC_UTF_8, &result, + NULL) == MJB_STATUS_OK) { mjb_result_free(&result); } size_t required = 0; - if(mjb_map_case_into(buffer, size, encoding, (mjb_map_case_type)(1 + (variant % 5)), - MJB_ENC_UTF_8, NULL, &required) == MJB_STATUS_OK && + if(mjb_map_case_into(buffer, size, encoding, malformed_policy, + (mjb_map_case_type)(1 + (variant % 5)), MJB_ENC_UTF_8, NULL, &required, + NULL) == MJB_STATUS_OK && required <= 4096) { char output[4096]; size_t capacity = (variant & 0x80) != 0 && required > 0 ? required - 1 : required; - fuzz_sink += (size_t)mjb_map_case_into(buffer, size, encoding, - (mjb_map_case_type)(1 + (variant % 5)), MJB_ENC_UTF_8, output, &capacity); + fuzz_sink += (size_t)mjb_map_case_into(buffer, size, encoding, malformed_policy, + (mjb_map_case_type)(1 + (variant % 5)), MJB_ENC_UTF_8, output, &capacity, NULL); } break; @@ -458,40 +463,44 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { case 5: { // Encoding conversion mjb_encoding output_encoding = encodings[(variant >> 1) % 6]; - if(mjb_convert_encoding(buffer, size, encoding, output_encoding, &result) == - MJB_STATUS_OK) { + mjb_malformed_policy policy = (mjb_malformed_policy)(variant % 3); + + if(mjb_convert_encoding(buffer, size, encoding, policy, output_encoding, &result, + NULL) == MJB_STATUS_OK) { mjb_result_free(&result); } size_t required = 0; - if(mjb_convert_encoding_into(buffer, size, encoding, output_encoding, NULL, - &required) == MJB_STATUS_OK && + if(mjb_convert_encoding_into(buffer, size, encoding, policy, output_encoding, NULL, + &required, NULL) == MJB_STATUS_OK && required <= 4096) { char output[4096]; size_t capacity = (variant & 0x80) != 0 && required > 0 ? required - 1 : required; - fuzz_sink += (size_t)mjb_convert_encoding_into(buffer, size, encoding, - output_encoding, output, &capacity); + fuzz_sink += (size_t)mjb_convert_encoding_into(buffer, size, encoding, policy, + output_encoding, output, &capacity, NULL); } break; } case 6: { // String filtering, all filter combinations - if(mjb_filter(buffer, size, encoding, (mjb_filter_flags)(variant & 0x1F), MJB_ENC_UTF_8, - &result) == MJB_STATUS_OK) { + mjb_malformed_policy policy = (mjb_malformed_policy)(variant % 3); + + if(mjb_filter(buffer, size, encoding, policy, (mjb_filter_flags)(variant & 0x1F), + MJB_ENC_UTF_8, &result, NULL) == MJB_STATUS_OK) { mjb_result_free(&result); } size_t required = 0; - if(mjb_filter_into(buffer, size, encoding, (mjb_filter_flags)(variant & 0x1F), - MJB_ENC_UTF_8, NULL, &required) == MJB_STATUS_OK && + if(mjb_filter_into(buffer, size, encoding, policy, (mjb_filter_flags)(variant & 0x1F), + MJB_ENC_UTF_8, NULL, &required, NULL) == MJB_STATUS_OK && required <= 4096) { char output[4096]; size_t capacity = (variant & 0x80) != 0 && required > 0 ? required - 1 : required; - fuzz_sink += (size_t)mjb_filter_into(buffer, size, encoding, - (mjb_filter_flags)(variant & 0x1F), MJB_ENC_UTF_8, output, &capacity); + fuzz_sink += (size_t)mjb_filter_into(buffer, size, encoding, policy, + (mjb_filter_flags)(variant & 0x1F), MJB_ENC_UTF_8, output, &capacity, NULL); } break; @@ -500,9 +509,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { case 7: { // Collation key mjb_collation_strength strength = (mjb_collation_strength)((variant >> 5) % 4); - if(mjb_collation_key(buffer, size, encoding, + if(mjb_collation_key(buffer, size, encoding, malformed_policy, (variant & 0x10) ? MJB_COLLATION_SHIFTED : MJB_COLLATION_NON_IGNORABLE, strength, - &result) == MJB_STATUS_OK) { + &result, NULL) == MJB_STATUS_OK) { mjb_result_free(&result); } @@ -511,13 +520,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { MJB_COLLATION_SHIFTED : MJB_COLLATION_NON_IGNORABLE; - if(mjb_collation_key_into(buffer, size, encoding, variable_weighting, strength, NULL, - &required) == MJB_STATUS_OK && + if(mjb_collation_key_into(buffer, size, encoding, malformed_policy, variable_weighting, + strength, NULL, &required, NULL) == MJB_STATUS_OK && required <= 4096) { char output[4096]; size_t capacity = required; fuzz_sink += (size_t)mjb_collation_key_into(buffer, size, encoding, - variable_weighting, strength, output, &capacity); + malformed_policy, variable_weighting, strength, output, &capacity, NULL); } break; @@ -534,7 +543,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { case 9: { // Segmentation: grapheme, word and width truncation, segment counts size_t segment_count = 0; - fuzz_sink += (size_t)mjb_codepoint_count(buffer, size, encoding, &segment_count); + fuzz_sink += (size_t)mjb_codepoint_count(buffer, size, encoding, MJB_MALFORMED_REPLACE, + &segment_count, NULL); fuzz_sink += segment_count; mjb_truncate_grapheme(buffer, size, encoding, variant); mjb_truncate_word(buffer, size, encoding, variant); @@ -543,19 +553,22 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { mjb_truncate_word_width(buffer, size, encoding, (mjb_terminal_width_profile)(variant % 2), variant); - fuzz_sink += (size_t)mjb_grapheme_count(buffer, size, encoding, &segment_count); + fuzz_sink += (size_t)mjb_grapheme_count(buffer, size, encoding, malformed_policy, + &segment_count, NULL); fuzz_sink += segment_count; - fuzz_sink += (size_t)mjb_sentence_count(buffer, size, encoding, &segment_count); + fuzz_sink += (size_t)mjb_sentence_count(buffer, size, encoding, malformed_policy, + &segment_count, NULL); fuzz_sink += segment_count; - fuzz_sink += (size_t)mjb_word_count(buffer, size, encoding, &segment_count); + fuzz_sink += (size_t)mjb_word_count(buffer, size, encoding, malformed_policy, + &segment_count, NULL); fuzz_sink += segment_count; break; } case 10: { // Terminal width size_t width = 0; - mjb_status status = mjb_terminal_width(buffer, size, encoding, - (mjb_terminal_width_profile)(variant % 2), &width); + mjb_status status = mjb_terminal_width(buffer, size, encoding, malformed_policy, + (mjb_terminal_width_profile)(variant % 2), &width, NULL); fuzz_sink += (size_t)status; if(status == MJB_STATUS_OK) { fuzz_sink += width; @@ -617,23 +630,24 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { case 16: // Raw boundary iterators and character callback API fuzz_boundary_iterators(buffer, size, encoding); fuzz_sink += (size_t)mjb_for_each_codepoint(buffer, size, encoding, - fuzz_next_codepoint); + MJB_MALFORMED_REPLACE, fuzz_next_codepoint, NULL); break; case 17: { // Identifier-oriented NFKC case folding - if(mjb_nfkc_casefold(buffer, size, encoding, MJB_ENC_UTF_8, &result) == MJB_STATUS_OK) { + if(mjb_nfkc_casefold(buffer, size, encoding, malformed_policy, MJB_ENC_UTF_8, &result, + NULL) == MJB_STATUS_OK) { mjb_result_free(&result); } size_t required = 0; - if(mjb_nfkc_casefold_into(buffer, size, encoding, MJB_ENC_UTF_8, NULL, &required) == - MJB_STATUS_OK && + if(mjb_nfkc_casefold_into(buffer, size, encoding, malformed_policy, MJB_ENC_UTF_8, NULL, + &required, NULL) == MJB_STATUS_OK && required <= 4096) { char output[4096]; size_t capacity = required; - fuzz_sink += (size_t)mjb_nfkc_casefold_into(buffer, size, encoding, MJB_ENC_UTF_8, - output, &capacity); + fuzz_sink += (size_t)mjb_nfkc_casefold_into(buffer, size, encoding, + malformed_policy, MJB_ENC_UTF_8, output, &capacity, NULL); } break; diff --git a/src/api/index.ts b/src/api/index.ts index f8598853..7a462c30 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -84,6 +84,14 @@ export enum Status { CALLBACK_STOPPED, NOT_FOUND, FEATURE_NOT_ENABLED, + END_OF_INPUT, +}; + +// mjb_malformed_policy +export enum MalformedPolicy { + STOP, + REPLACE, + SKIP, }; // mjb_idna_error @@ -324,6 +332,7 @@ export type TextInputOptions = { encoding?: Encoding; additionalEncoding?: Encoding; outputEncoding?: Encoding; + malformedPolicy?: MalformedPolicy; }; // Used for preRun and postRun callbacks @@ -446,7 +455,8 @@ export class Mojibake { } // mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_encoding encoding, - // mjb_normalization form, mjb_encoding output_encoding, mjb_result *result) + // mjb_malformed_policy malformed_policy, mjb_normalization form, + // mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic) normalize(input: MojibakeInput, form = Normalization.NFC, options: TextInputOptions = {}): Result | null { const wasmInput = this.copyInput(input, options.encoding); @@ -456,7 +466,8 @@ export class Mojibake { try { const status = this.module._mjb_normalize(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, form, outputEncoding, resultPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, form, + outputEncoding, resultPtr, 0); if(status !== Status.OK) { return null; @@ -550,7 +561,8 @@ export class Mojibake { } // mjb_status mjb_nfkc_casefold(const char *buffer, size_t byte_length, mjb_encoding encoding, - // mjb_encoding output_encoding, mjb_result *result) + // mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, + // mjb_result *result, mjb_diagnostic *diagnostic) nfkcCasefold(input: MojibakeInput, options: TextInputOptions = {}): Result | null { const wasmInput = this.copyInput(input, options.encoding); const outputEncoding = this.resolveEncoding(options.outputEncoding ?? wasmInput.encoding); @@ -559,7 +571,8 @@ export class Mojibake { try { const status = this.module._mjb_nfkc_casefold(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, outputEncoding, resultPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, + outputEncoding, resultPtr, 0); if(status !== Status.OK) { return null; @@ -616,7 +629,7 @@ export class Mojibake { try { const status = this.module._mjb_for_each_codepoint(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, 0); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, 0, 0); if(status !== Status.OK) { return null; @@ -635,7 +648,8 @@ export class Mojibake { } // mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_encoding encoding, - // mjb_filter_flags filters, mjb_encoding output_encoding, mjb_result *result) + // mjb_malformed_policy malformed_policy, mjb_filter_flags filters, + // mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic) filter(input: MojibakeInput, filters = FilterFlags.NONE, options: TextInputOptions = {}): Result | null { const wasmInput = this.copyInput(input, options.encoding); @@ -645,7 +659,8 @@ export class Mojibake { try { const status = this.module._mjb_filter(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, filters, outputEncoding, resultPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, filters, + outputEncoding, resultPtr, 0); if(status !== Status.OK) { return null; @@ -845,7 +860,8 @@ export class Mojibake { } // mjb_status mjb_convert_encoding(const char *buffer, size_t byte_length, mjb_encoding encoding, - // mjb_encoding output_encoding, mjb_result *result) + // mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, mjb_result *result, + // mjb_diagnostic *diagnostic) convertEncoding(input: MojibakeInput, outputEncoding = Encoding.UTF_8, options: TextInputOptions = {}): Result | null { const wasmInput = this.copyInput(input, options.encoding); @@ -855,7 +871,8 @@ export class Mojibake { try { const status = this.module._mjb_convert_encoding(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, outputEncoding, resultPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, outputEncoding, + resultPtr, 0); if(status !== Status.OK) { return null; @@ -875,14 +892,14 @@ export class Mojibake { } // mjb_status mjb_codepoint_count(const char *buffer, size_t byte_length, mjb_encoding encoding, - // size_t *count) + // mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic) codepointCount(input: MojibakeInput, options: TextInputOptions = {}): number | null { const wasmInput = this.copyInput(input, options.encoding); const countPtr = this.malloc(4); try { const status = this.module._mjb_codepoint_count(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, countPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, countPtr, 0); if(status === Status.OK) { return this.module.HEAP32[countPtr / 4]; @@ -943,8 +960,9 @@ export class Mojibake { } // mjb_status mjb_collation_key(const char *buffer, size_t byte_length, mjb_encoding encoding, + // mjb_malformed_policy malformed_policy, // mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, - // mjb_result *result) + // mjb_result *result, mjb_diagnostic *diagnostic) collationKey(input: MojibakeInput, variableWeighting = CollationVariableWeighting.NON_IGNORABLE, strength = CollationStrength.TERTIARY, @@ -955,7 +973,8 @@ export class Mojibake { try { const status = this.module._mjb_collation_key(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, variableWeighting, strength, resultPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, + variableWeighting, strength, resultPtr, 0); if(status !== Status.OK) { return null; @@ -980,7 +999,8 @@ export class Mojibake { } // mjb_status mjb_map_case(const char *buffer, size_t byte_length, mjb_encoding encoding, - // mjb_map_case_type type, mjb_encoding output_encoding, mjb_result *result) + // mjb_malformed_policy malformed_policy, mjb_map_case_type type, + // mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic) mapCase(input: MojibakeInput, type: CaseType, options: TextInputOptions = {}): Result | null { const wasmInput = this.copyInput(input, options.encoding); const outputEncoding = this.resolveEncoding(options.outputEncoding ?? wasmInput.encoding); @@ -989,7 +1009,7 @@ export class Mojibake { try { const status = this.module._mjb_map_case(wasmInput.ptr, wasmInput.size, wasmInput.encoding, - type, outputEncoding, resultPtr); + options.malformedPolicy ?? MalformedPolicy.STOP, type, outputEncoding, resultPtr, 0); if(status !== Status.OK) { return null; @@ -1168,14 +1188,14 @@ export class Mojibake { } // mjb_status mjb_grapheme_count(const char *buffer, size_t byte_length, mjb_encoding encoding, - // size_t *count) + // mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic) graphemeCount(input: MojibakeInput, options: TextInputOptions = {}): number | null { const wasmInput = this.copyInput(input, options.encoding); const countPtr = this.malloc(4); try { const status = this.module._mjb_grapheme_count(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, countPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, countPtr, 0); if(status === Status.OK) { return this.module.HEAP32[countPtr / 4]; @@ -1189,14 +1209,14 @@ export class Mojibake { } // mjb_status mjb_sentence_count(const char *buffer, size_t byte_length, mjb_encoding encoding, - // size_t *count) + // mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic) sentenceCount(input: MojibakeInput, options: TextInputOptions = {}): number | null { const wasmInput = this.copyInput(input, options.encoding); const countPtr = this.malloc(4); try { const status = this.module._mjb_sentence_count(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, countPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, countPtr, 0); if(status === Status.OK) { return this.module.HEAP32[countPtr / 4]; @@ -1210,14 +1230,14 @@ export class Mojibake { } // mjb_status mjb_word_count(const char *buffer, size_t byte_length, mjb_encoding encoding, - // size_t *count) + // mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic) wordCount(input: MojibakeInput, options: TextInputOptions = {}): number | null { const wasmInput = this.copyInput(input, options.encoding); const countPtr = this.malloc(4); try { const status = this.module._mjb_word_count(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, countPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, countPtr, 0); if(status === Status.OK) { return this.module.HEAP32[countPtr / 4]; @@ -1542,7 +1562,8 @@ export class Mojibake { } // mjb_status mjb_terminal_width(const char *buffer, size_t byte_length, mjb_encoding encoding, - // mjb_terminal_width_profile profile, size_t *width); + // mjb_malformed_policy malformed_policy, mjb_terminal_width_profile profile, + // size_t *width, mjb_diagnostic *diagnostic); terminalWidth(input: MojibakeInput, profile = TerminalWidthProfile.NARROW, options: TextInputOptions = {}): number | null { const wasmInput = this.copyInput(input, options.encoding); @@ -1550,7 +1571,8 @@ export class Mojibake { try { const status = this.module._mjb_terminal_width(wasmInput.ptr, wasmInput.size, - wasmInput.encoding, profile, widthPtr); + wasmInput.encoding, options.malformedPolicy ?? MalformedPolicy.STOP, + profile, widthPtr, 0); if(status === Status.OK) { return this.module.HEAP32[widthPtr / 4]; diff --git a/src/api/mojibake.d.ts b/src/api/mojibake.d.ts index bf5620f4..3cf6a10e 100644 --- a/src/api/mojibake.d.ts +++ b/src/api/mojibake.d.ts @@ -24,11 +24,11 @@ export type MojibakeWasmModule = { // Return the codepoint character. _mjb_codepoint_info: (codepoint: Codepoint, character: Pointer) => number; // Normalize a string to NFC/NFKC/NFD/NFKD form. - _mjb_normalize: (buffer: Pointer, byte_length: number, encoding: number, form: number, output_encoding: number, result: Pointer) => number; + _mjb_normalize: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, form: number, output_encoding: number, result: Pointer, diagnostic: Pointer) => number; // Filter a string with the selected mjb_filter_flags. - _mjb_filter: (buffer: Pointer, byte_length: number, encoding: number, filters: number, output_encoding: number, result: Pointer) => number; + _mjb_filter: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, filters: number, output_encoding: number, result: Pointer, diagnostic: Pointer) => number; // Apply the Unicode NFKC_Casefold transform to a string. - _mjb_nfkc_casefold: (buffer: Pointer, byte_length: number, encoding: number, output_encoding: number, result: Pointer) => number; + _mjb_nfkc_casefold: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, output_encoding: number, result: Pointer, diagnostic: Pointer) => number; // Convert a domain name to its UTS #46 nontransitional ASCII form. _mjb_idna_to_ascii: (buffer: Pointer, byte_length: number, encoding: number, output_encoding: number, info: Pointer, result: Pointer) => number; // Convert a domain name to its UTS #46 nontransitional Unicode form. @@ -44,9 +44,9 @@ export type MojibakeWasmModule = { // Return true if the string is encoded in UTF-16BE or UTF-16LE. _mjb_is_utf16: (buffer: Pointer, byte_length: number) => boolean; // Count the codepoints in a string. - _mjb_codepoint_count: (buffer: Pointer, byte_length: number, encoding: number, count: Pointer) => number; + _mjb_codepoint_count: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, count: Pointer, diagnostic: Pointer) => number; // Run a callback for each codepoint of a string. - _mjb_for_each_codepoint: (buffer: Pointer, byte_length: number, encoding: number, callback: number) => number; + _mjb_for_each_codepoint: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, callback: number, diagnostic: Pointer) => number; // Return the value of a binary Unicode property. _mjb_codepoint_property_binary: (codepoint: Codepoint, property: number, value: Pointer) => number; // Return the value of an enumerated or integer Unicode property. @@ -62,15 +62,15 @@ export type MojibakeWasmModule = { // Encode a codepoint to a string. _mjb_codepoint_encode: (codepoint: Codepoint, buffer: Pointer, byte_length: number, encoding: number) => number; // Convert from one encoding to another. - _mjb_convert_encoding: (buffer: Pointer, byte_length: number, encoding: number, output_encoding: number, result: Pointer) => number; + _mjb_convert_encoding: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, output_encoding: number, result: Pointer, diagnostic: Pointer) => number; // Compare two strings using a Unicode caseless matching relation. _mjb_caseless_match: (s1: Pointer, s1_byte_length: number, s1_encoding: number, s2: Pointer, s2_byte_length: number, s2_encoding: number, mode: number, matches: Pointer) => number; // Compare two strings using UCA. _mjb_collation_compare: (s1: Pointer, s1_byte_length: number, s1_encoding: number, s2: Pointer, s2_byte_length: number, s2_encoding: number, variable_weighting: number, strength: number, order: Pointer) => number; // Generate a UCA sort key for a string. - _mjb_collation_key: (buffer: Pointer, byte_length: number, encoding: number, variable_weighting: number, strength: number, result: Pointer) => number; + _mjb_collation_key: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, variable_weighting: number, strength: number, result: Pointer, diagnostic: Pointer) => number; // Change string case. - _mjb_map_case: (buffer: Pointer, byte_length: number, encoding: number, type: number, output_encoding: number, result: Pointer) => number; + _mjb_map_case: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, type: number, output_encoding: number, result: Pointer, diagnostic: Pointer) => number; // Return true if the codepoint is valid. _mjb_codepoint_is_valid: (codepoint: Codepoint) => boolean; // Return true if the codepoint is graphic. @@ -102,19 +102,19 @@ export type MojibakeWasmModule = { // Sentence boundaries breaking. _mjb_next_sentence_break: (buffer: Pointer, byte_length: number, encoding: number, state: Pointer) => number; // Count the sentence segments in a string. - _mjb_sentence_count: (buffer: Pointer, byte_length: number, encoding: number, count: Pointer) => number; + _mjb_sentence_count: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, count: Pointer, diagnostic: Pointer) => number; // Grapheme cluster breaking. _mjb_next_grapheme_break: (buffer: Pointer, byte_length: number, encoding: number, state: Pointer) => number; // Return the number of bytes that form the first `max_graphemes` grapheme cluster segments. _mjb_truncate_grapheme: (buffer: Pointer, byte_length: number, encoding: number, max_graphemes: number) => number; // Count the extended grapheme clusters in a string. - _mjb_grapheme_count: (buffer: Pointer, byte_length: number, encoding: number, count: Pointer) => number; + _mjb_grapheme_count: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, count: Pointer, diagnostic: Pointer) => number; // Return the number of bytes whose grapheme clusters fit within max_columns terminal cells. _mjb_truncate_grapheme_width: (buffer: Pointer, byte_length: number, encoding: number, profile: number, max_columns: number) => number; // Return the number of bytes that form the first max_segments word-break segments. _mjb_truncate_word: (buffer: Pointer, byte_length: number, encoding: number, max_segments: number) => number; // Count the word-like segments in a string. - _mjb_word_count: (buffer: Pointer, byte_length: number, encoding: number, count: Pointer) => number; + _mjb_word_count: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, count: Pointer, diagnostic: Pointer) => number; // Return the number of bytes whose word-break segments fit within max_columns terminal cells. _mjb_truncate_word_width: (buffer: Pointer, byte_length: number, encoding: number, profile: number, max_columns: number) => number; // Resolve bidirectional text (TR9) for a paragraph. @@ -172,7 +172,7 @@ export type MojibakeWasmModule = { // Return the east asian width of a codepoint. _mjb_codepoint_east_asian_width: (codepoint: Codepoint, width: Pointer) => number; // Return the estimated terminal-cell width of printable, single-line text. - _mjb_terminal_width: (buffer: Pointer, byte_length: number, encoding: number, profile: number, width: Pointer) => number; + _mjb_terminal_width: (buffer: Pointer, byte_length: number, encoding: number, malformed_policy: number, profile: number, width: Pointer, diagnostic: Pointer) => number; // Parse a BCP 47 language tag. _mjb_locale_parse: (id: Pointer, byte_length: number, encoding: number, locale: Pointer) => number; // Set the current process-global locale. diff --git a/src/api/tests/index.ts b/src/api/tests/index.ts index bdc02bdf..be10d826 100644 --- a/src/api/tests/index.ts +++ b/src/api/tests/index.ts @@ -20,6 +20,7 @@ import { FilterFlags, IdnaError, Locale, + MalformedPolicy, Mojibake, Normalization, Plane, @@ -66,6 +67,21 @@ ATT_ASSERT(mojibake.filter('hello world', FilterFlags.COLLAPSE_SPACES | FilterFlags.CONTROLS)?.output, 'hello world', 'filter'); ATT_ASSERT(mojibake.filter('a\u0300\u0301\u0302\u0303\u0304', FilterFlags.LIMIT_COMBINING)?.output, 'a\u0300\u0301\u0302\u0303', 'filter LIMIT_COMBINING'); +const malformedText = new Uint8Array([0x41, 0x80, 0x42]); +ATT_ASSERT(mojibake.normalize(malformedText), null, 'normalize malformed stop policy'); +ATT_ASSERT(mojibake.normalize(malformedText, Normalization.NFC, + { malformedPolicy: MalformedPolicy.REPLACE })?.output, 'A\uFFFDB', + 'normalize malformed replace policy'); +ATT_ASSERT(mojibake.normalize(malformedText, Normalization.NFC, + { malformedPolicy: MalformedPolicy.SKIP })?.output, 'AB', + 'normalize malformed skip policy'); +ATT_ASSERT(mojibake.filter(malformedText), null, 'filter malformed stop policy'); +ATT_ASSERT(mojibake.filter(malformedText, FilterFlags.NONE, + { malformedPolicy: MalformedPolicy.REPLACE })?.output, 'A\uFFFDB', + 'filter malformed replace policy'); +ATT_ASSERT(mojibake.filter(malformedText, FilterFlags.NONE, + { malformedPolicy: MalformedPolicy.SKIP })?.output, 'AB', + 'filter malformed skip policy'); ATT_ASSERT(mojibake.codepointPropertyBinary(0x41, Property.ALPHABETIC), true, 'codepointPropertyBinary true'); ATT_ASSERT(mojibake.codepointPropertyBinary(0x20, Property.ALPHABETIC), false, @@ -95,8 +111,20 @@ ATT_ASSERT(mojibake.isUTF16(new Uint8Array([0x00, 0x48, 0x00, 0x69])), true, 'is ATT_ASSERT(mojibake.isASCII('Hello'), true, 'isASCII'); ATT_ASSERT(mojibake.codepointEncode(0x41)?.output, 'A', 'codepointEncode'); ATT_ASSERT(mojibake.convertEncoding('A', Encoding.UTF_16LE)?.output, 'A', 'convertEncoding'); +ATT_ASSERT(mojibake.convertEncoding(malformedText), null, 'convertEncoding malformed stop policy'); +ATT_ASSERT(mojibake.convertEncoding(malformedText, Encoding.UTF_8, + { malformedPolicy: MalformedPolicy.REPLACE })?.output, 'A\uFFFDB', + 'convertEncoding malformed replace policy'); +ATT_ASSERT(mojibake.convertEncoding(malformedText, Encoding.UTF_8, + { malformedPolicy: MalformedPolicy.SKIP })?.output, 'AB', + 'convertEncoding malformed skip policy'); ATT_ASSERT(mojibake.codepointCount('H\u00E9ll\u00F6'), 5, 'codepointCount'); ATT_ASSERT(mojibake.codepointCount(''), 0, 'codepointCount empty'); +ATT_ASSERT(mojibake.codepointCount(malformedText), null, 'codepointCount malformed stop policy'); +ATT_ASSERT(mojibake.codepointCount(malformedText, + { malformedPolicy: MalformedPolicy.REPLACE }), 3, 'codepointCount malformed replace policy'); +ATT_ASSERT(mojibake.codepointCount(malformedText, + { malformedPolicy: MalformedPolicy.SKIP }), 2, 'codepointCount malformed skip policy'); ATT_ASSERT(mojibake.caselessMatch('Straße', 'STRASSE'), true, 'caselessMatch'); ATT_ASSERT(mojibake.caselessMatch('\u00C5', 'A\u030A', CaselessMode.UNNORMALIZED), false, 'caselessMatch unnormalized'); @@ -151,9 +179,15 @@ ATT_ASSERT(mojibake.collationKey('A', CollationVariableWeighting.NON_IGNORABLE, CollationStrength.SECONDARY), mojibake.collationKey('a', CollationVariableWeighting.NON_IGNORABLE, CollationStrength.SECONDARY), 'collationKey secondary ignores case'); +ATT_ASSERT(mojibake.collationKey(malformedText, CollationVariableWeighting.NON_IGNORABLE, + CollationStrength.TERTIARY, { malformedPolicy: MalformedPolicy.SKIP }), +mojibake.collationKey('AB'), 'collationKey malformed skip policy'); ATT_ASSERT(mojibake.mapCase('hello', CaseType.UPPER)?.output, 'HELLO', 'mapCase'); ATT_ASSERT(mojibake.mapCase('\u13A0', CaseType.CASEFOLD)?.output, '\u13A0', 'mapCase casefold uppercase Cherokee'); +ATT_ASSERT(mojibake.mapCase(malformedText, CaseType.LOWER, + { malformedPolicy: MalformedPolicy.REPLACE })?.output, 'a\uFFFDb', + 'mapCase malformed replace policy'); ATT_ASSERT(mojibake.codepointIsValid(0x41), true, 'codepointIsValid'); ATT_ASSERT(mojibake.codepointIsGraphic(0x23), true, 'codepointIsGraphic'); ATT_ASSERT(mojibake.codepointIsCombining(0x0300), true, 'codepointIsCombining'); @@ -171,6 +205,9 @@ ATT_ASSERT(mojibake.codepointNumericValue(0x31), { decimal: 1, digit: 1, numeric 'codepointNumericValue'); ATT_ASSERT(mojibake.codepointBlock(0x41)?.id, Block.BASIC_LATIN, 'codepointBlock'); ATT_ASSERT(mojibake.nfkcCasefold('Straße\u00AD')?.output, 'strasse', 'nfkcCasefold'); +ATT_ASSERT(mojibake.nfkcCasefold(malformedText, + { malformedPolicy: MalformedPolicy.SKIP })?.output, 'ab', + 'nfkcCasefold malformed skip policy'); ATT_ASSERT(mojibake.nextLineBreak('A'), [BreakType.ALLOWED], 'nextLineBreak'); ATT_ASSERT(mojibake.nextWordBreak('A'), [BreakType.ALLOWED], 'nextWordBreak'); ATT_ASSERT(mojibake.nextSentenceBreak('A'), [BreakType.ALLOWED], 'nextSentenceBreak'); @@ -183,12 +220,21 @@ ATT_ASSERT(mojibake.graphemeCount('ABC'), 3, 'graphemeCount'); ATT_ASSERT(mojibake.graphemeCount(''), 0, 'graphemeCount empty'); ATT_ASSERT(mojibake.graphemeCount('🇮🇹'), 1, 'graphemeCount flag emoji'); ATT_ASSERT(mojibake.graphemeCount('👨‍👩‍👦'), 1, 'graphemeCount ZWJ sequence'); +ATT_ASSERT(mojibake.graphemeCount(malformedText, + { malformedPolicy: MalformedPolicy.REPLACE }), 3, + 'graphemeCount malformed replace policy'); ATT_ASSERT(mojibake.sentenceCount('Hello. How are you? Fine!'), 3, 'sentenceCount'); ATT_ASSERT(mojibake.sentenceCount(''), 0, 'sentenceCount empty'); +ATT_ASSERT(mojibake.sentenceCount(malformedText, + { malformedPolicy: MalformedPolicy.SKIP }), 1, + 'sentenceCount malformed skip policy'); ATT_ASSERT(mojibake.wordCount('Hello, world! It works.'), 4, 'wordCount'); ATT_ASSERT(mojibake.wordCount('state-of-the-art'), 4, 'wordCount hyphenated'); ATT_ASSERT(mojibake.wordCount('...'), 0, 'wordCount punctuation only'); ATT_ASSERT(mojibake.wordCount(''), 0, 'wordCount empty'); +ATT_ASSERT(mojibake.wordCount(malformedText, + { malformedPolicy: MalformedPolicy.REPLACE }), 2, + 'wordCount malformed replace policy'); ATT_ASSERT(mojibake.truncateGraphemeWidth('ABC', TerminalWidthProfile.NARROW, 2), 2, 'truncateGraphemeWidth'); ATT_ASSERT(mojibake.bidiResolve('ABC', Direction.AUTO)?.direction, Direction.LTR, 'bidiResolve'); @@ -230,6 +276,9 @@ ATT_ASSERT(mojibake.codepointEastAsianWidth(0x20), EastAsianWidth.NARROW, ATT_ASSERT(mojibake.terminalWidth('Hello'), 5, 'terminalWidth'); ATT_ASSERT(mojibake.terminalWidth('👨🏻‍❤️‍💋‍👨🏻'), 2, 'terminalWidth emoji sequence'); ATT_ASSERT(mojibake.terminalWidth('line\nbreak'), null, 'terminalWidth rejects controls'); +ATT_ASSERT(mojibake.terminalWidth(malformedText, TerminalWidthProfile.NARROW, + { malformedPolicy: MalformedPolicy.SKIP }), 2, + 'terminalWidth malformed skip policy'); ATT_ASSERT(mojibake.localeParse('sr-Latn-RS').region, 'RS', 'localeParse'); ATT_ASSERT(mojibake.setLocale(Locale.IT), true, 'setLocale'); ATT_ASSERT(mojibake.getLocale(), Locale.IT, 'getLocale'); diff --git a/src/bidi.c b/src/bidi.c index 324af5ec..01be3ea0 100644 --- a/src/bidi.c +++ b/src/bidi.c @@ -329,15 +329,16 @@ static bool bidi_query(mjb_codepoint cp, mjb_bidi_class *out_class, bool *out_mi } // Pass 1: decode string + build working array + P2/P3 paragraph level. -static size_t pass1_decode(const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_bidi_work *work, size_t capacity, uint8_t *out_level, mjb_direction base_dir) { +static mjb_status pass1_decode(const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_bidi_work *work, size_t capacity, uint8_t *out_level, mjb_direction base_dir, + size_t *out_count) { uint8_t state = MJB_UTF_ACCEPT; bool in_error = false; mjb_codepoint cp = 0; size_t count = 0; size_t i = 0; - while(i < byte_length && count < capacity) { + while(count < capacity) { size_t byte_offset = i; mjb_decode_result dr = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, &cp, &in_error); @@ -350,6 +351,10 @@ static size_t pass1_decode(const char *buffer, size_t byte_length, mjb_encoding continue; } + if(dr == MJB_DECODE_ERROR) { + return MJB_STATUS_MALFORMED_INPUT; + } + mjb_bidi_class bc; bool mirrored; @@ -393,7 +398,9 @@ static size_t pass1_decode(const char *buffer, size_t byte_length, mjb_encoding } } - return count; + *out_count = count; + + return MJB_STATUS_OK; } // Pass 2: X rules, explicit levels @@ -1182,6 +1189,10 @@ MJB_EXPORT mjb_status mjb_bidi_resolve(const char *buffer, size_t byte_length, return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -1197,6 +1208,12 @@ MJB_EXPORT mjb_status mjb_bidi_resolve(const char *buffer, size_t byte_length, return MJB_STATUS_OK; } + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); + + if(status != MJB_STATUS_OK) { + return status; + } + // Upper bound: size bytes cannot produce more than size codepoints. if(byte_length > SIZE_MAX / sizeof(mjb_bidi_work)) { return MJB_STATUS_OVERFLOW; @@ -1211,8 +1228,15 @@ MJB_EXPORT mjb_status mjb_bidi_resolve(const char *buffer, size_t byte_length, uint8_t para_level = 0; // Pass 1. - size_t count = pass1_decode(buffer, byte_length, encoding, work, byte_length, ¶_level, - direction); + size_t count = 0; + status = pass1_decode(buffer, byte_length, encoding, work, byte_length, ¶_level, direction, + &count); + + if(status != MJB_STATUS_OK) { + mjb_free(work); + + return status; + } if(count == 0) { mjb_free(work); diff --git a/src/break-sentence.c b/src/break-sentence.c index 5b6e9284..8c583c3e 100644 --- a/src/break-sentence.c +++ b/src/break-sentence.c @@ -89,6 +89,7 @@ MJB_EXPORT mjb_break_type mjb_next_sentence_break(const char *buffer, size_t byt state->previous_codepoint = MJB_CODEPOINT_NOT_VALID; state->current_codepoint = MJB_CODEPOINT_NOT_VALID; state->in_error = false; + state->had_error = false; state->sb5_merged = false; state->in_sat = false; state->sat_has_sp = false; @@ -101,6 +102,10 @@ MJB_EXPORT mjb_break_type mjb_next_sentence_break(const char *buffer, size_t byt if(state->index == byte_length) { // Reached end of string. + if(mjb_utf_state_is_incomplete(state->state)) { + state->had_error = true; + } + ++state->index; // SB2 Any ÷ eot @@ -118,6 +123,10 @@ MJB_EXPORT mjb_break_type mjb_next_sentence_break(const char *buffer, size_t byt mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state->state, &state->index, encoding, &codepoint, &state->in_error); + if(decode_status == MJB_DECODE_ERROR) { + state->had_error = true; + } + if(decode_status == MJB_DECODE_END) { mjb_mark_decode_terminated(&state->state, &state->index, &state->current_codepoint, encoding); @@ -294,20 +303,63 @@ MJB_EXPORT mjb_break_type mjb_next_sentence_break(const char *buffer, size_t byt return MJB_BT_NO_BREAK; } + if(mjb_utf_state_is_incomplete(state->state)) { + state->had_error = true; + } + ++state->index; // SB2 Any ÷ eot. return MJB_BT_ALLOWED; } +static mjb_status mjb_sentence_count_process(const char *buffer, size_t byte_length, + mjb_encoding encoding, size_t *count) { + if(byte_length == 0) { + *count = 0; + + return MJB_STATUS_OK; + } + + mjb_next_sentence_state state; + state.index = 0; + + mjb_break_type bt; + size_t segment_count = 0; + + while((bt = mjb_next_sentence_break(buffer, byte_length, encoding, &state)) != MJB_BT_NOT_SET) { + if(bt != MJB_BT_NO_BREAK) { + ++segment_count; + } + } + + if(state.had_error) { + return MJB_STATUS_MALFORMED_INPUT; + } + + *count = segment_count; + + return MJB_STATUS_OK; +} + // Count the sentence segments in a string. MJB_EXPORT mjb_status mjb_sentence_count(const char *buffer, size_t byte_length, - mjb_encoding encoding, size_t *count) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, + mjb_diagnostic *diagnostic) { if(count == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } *count = 0; + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } if(byte_length == 0) { return MJB_STATUS_OK; @@ -317,31 +369,31 @@ MJB_EXPORT mjb_status mjb_sentence_count(const char *buffer, size_t byte_length, return MJB_STATUS_INVALID_ARGUMENT; } - if(!mjb_encoding_is_valid_input(encoding)) { - return MJB_STATUS_INVALID_ENCODING; - } - mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK || byte_length == 0) { return status; } - mjb_next_sentence_state state; - state.index = 0; + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); - mjb_break_type bt; - size_t segment_count = 0; + if(status != MJB_STATUS_OK) { + return status; + } - while((bt = mjb_next_sentence_break(buffer, byte_length, encoding, &state)) != MJB_BT_NOT_SET) { - if(bt == MJB_BT_NO_BREAK) { - continue; - } + status = mjb_sentence_count_process(buffer, byte_length, encoding, count); + mjb_result sanitized = { NULL, 0, false }; + + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); - ++segment_count; + if(status == MJB_STATUS_OK) { + status = mjb_sentence_count_process(buffer, byte_length, encoding, count); + } } - *count = segment_count; + mjb_result_free(&sanitized); - return MJB_STATUS_OK; + return status; } diff --git a/src/break-word.c b/src/break-word.c index 25879a78..f0b9bc07 100644 --- a/src/break-word.c +++ b/src/break-word.c @@ -71,6 +71,7 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le state->current_codepoint = MJB_CODEPOINT_NOT_VALID; state->prev_prev_wbp = MJB_WBP_NOT_SET; state->in_error = false; + state->had_error = false; state->ri_count = 0; state->wb4_merged = false; state->zwj_pending = false; @@ -83,6 +84,10 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le if(state->index == byte_length) { // Reached end of string. + if(mjb_utf_state_is_incomplete(state->state)) { + state->had_error = true; + } + ++state->index; // WB2 Any ÷ eot @@ -100,6 +105,10 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state->state, &state->index, encoding, &codepoint, &state->in_error); + if(decode_status == MJB_DECODE_ERROR) { + state->had_error = true; + } + if(decode_status == MJB_DECODE_END) { mjb_mark_decode_terminated(&state->state, &state->index, &state->current_codepoint, encoding); @@ -344,6 +353,10 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le return MJB_BT_ALLOWED; } + if(mjb_utf_state_is_incomplete(state->state)) { + state->had_error = true; + } + ++state->index; return MJB_BT_ALLOWED; @@ -420,17 +433,66 @@ static bool mjb_segment_is_word_like(const char *buffer, size_t byte_length, return false; } +static mjb_status mjb_word_count_process(const char *buffer, size_t byte_length, + mjb_encoding encoding, size_t *count) { + if(byte_length == 0) { + *count = 0; + + return MJB_STATUS_OK; + } + + mjb_next_word_state state; + state.index = 0; + + mjb_break_type bt; + size_t word_count = 0; + size_t last_break = 0; + + while((bt = mjb_next_word_break(buffer, byte_length, encoding, &state)) != MJB_BT_NOT_SET) { + if(bt == MJB_BT_NO_BREAK) { + continue; + } + + size_t break_pos = mjb_monotonic_boundary_position(state.index, byte_length, + state.current_codepoint, encoding, state.state == MJB_UTF_TERMINATED, last_break); + + if(break_pos > last_break && + mjb_segment_is_word_like(buffer + last_break, break_pos - last_break, encoding)) { + ++word_count; + } + + last_break = break_pos; + } + + if(state.had_error) { + return MJB_STATUS_MALFORMED_INPUT; + } + + *count = word_count; + + return MJB_STATUS_OK; +} + // Count the word-break segments that contain at least one alphabetic or numeric character. // Scripts that other implementations segment by dictionary lookup, such as Chinese, Japanese, // Thai, Lao, Khmer, and Burmese, count roughly one word per character: Mojibake does not use // frequency dictionaries to keep the size of the library small. MJB_EXPORT mjb_status mjb_word_count(const char *buffer, size_t byte_length, mjb_encoding encoding, - size_t *count) { + mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic) { if(count == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } *count = 0; + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } if(byte_length == 0) { return MJB_STATUS_OK; @@ -440,42 +502,33 @@ MJB_EXPORT mjb_status mjb_word_count(const char *buffer, size_t byte_length, mjb return MJB_STATUS_INVALID_ARGUMENT; } - if(!mjb_encoding_is_valid_input(encoding)) { - return MJB_STATUS_INVALID_ENCODING; - } - mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK || byte_length == 0) { return status; } - mjb_next_word_state state; - state.index = 0; + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); - mjb_break_type bt; - size_t word_count = 0; - size_t last_break = 0; + if(status != MJB_STATUS_OK) { + return status; + } - while((bt = mjb_next_word_break(buffer, byte_length, encoding, &state)) != MJB_BT_NOT_SET) { - if(bt == MJB_BT_NO_BREAK) { - continue; - } + status = mjb_word_count_process(buffer, byte_length, encoding, count); + mjb_result sanitized = { NULL, 0, false }; - size_t break_pos = mjb_monotonic_boundary_position(state.index, byte_length, - state.current_codepoint, encoding, state.state == MJB_UTF_TERMINATED, last_break); + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); - if(break_pos > last_break && - mjb_segment_is_word_like(buffer + last_break, break_pos - last_break, encoding)) { - ++word_count; + if(status == MJB_STATUS_OK) { + status = mjb_word_count_process(buffer, byte_length, encoding, count); } - - last_break = break_pos; } - *count = word_count; + mjb_result_free(&sanitized); - return MJB_STATUS_OK; + return status; } // Return the number of bytes whose word-break segments fit within max_columns terminal cells. @@ -506,8 +559,8 @@ MJB_EXPORT size_t mjb_truncate_word_width(const char *buffer, size_t byte_length state.current_codepoint, encoding, state.state == MJB_UTF_TERMINATED, prev_break); size_t segment_width = 0; - if(mjb_terminal_width(buffer + prev_break, break_pos - prev_break, encoding, profile, - &segment_width) != MJB_STATUS_OK) { + if(mjb_terminal_width(buffer + prev_break, break_pos - prev_break, encoding, + MJB_MALFORMED_STOP, profile, &segment_width, NULL) != MJB_STATUS_OK) { return prev_break; } diff --git a/src/case.c b/src/case.c index 666d91fd..93aaf82c 100644 --- a/src/case.c +++ b/src/case.c @@ -346,7 +346,7 @@ static mjb_status mjb_titlecase_process(const char *buffer, size_t byte_length, &word_state, 0); bool segment_has_cased = false; - for(size_t i = 0; i < byte_length;) { + for(size_t i = 0;;) { size_t codepoint_start = i; // Find next codepoint. @@ -361,6 +361,10 @@ static mjb_status mjb_titlecase_process(const char *buffer, size_t byte_length, continue; } + if(decode_status == MJB_DECODE_ERROR) { + return MJB_STATUS_MALFORMED_INPUT; + } + while(codepoint_start >= segment_end && segment_end < byte_length) { segment_has_cased = false; segment_end = mjb_titlecase_next_word_boundary(buffer, byte_length, encoding, @@ -481,7 +485,7 @@ static mjb_status mjb_map_case_process(const char *buffer, size_t byte_length, bool track_context = type == MJB_CASE_LOWER || locale_sensitive; mjb_map_case_context context = { false, false, false, locale_sensitive }; - for(size_t i = 0; i < byte_length;) { + for(size_t i = 0;;) { // Find next codepoint. mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, &codepoint, &in_error); @@ -494,6 +498,10 @@ static mjb_status mjb_map_case_process(const char *buffer, size_t byte_length, continue; } + if(decode_status == MJB_DECODE_ERROR) { + return MJB_STATUS_MALFORMED_INPUT; + } + if(folding) { // Turkic (T) foldings [CaseFolding.txt]: in tr/az, I folds to ı and İ to i, in // both full and simple folding. @@ -720,16 +728,24 @@ static mjb_status mjb_map_case_result(const char *buffer, size_t byte_length, mj } MJB_EXPORT mjb_status mjb_map_case(const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_map_case_type type, mjb_encoding output_encoding, mjb_result *result) { - if(result == NULL || (buffer == NULL && byte_length > 0)) { + mjb_malformed_policy malformed_policy, mjb_map_case_type type, mjb_encoding output_encoding, + mjb_result *result, mjb_diagnostic *diagnostic) { + if(result == NULL || (buffer == NULL && byte_length > 0) || + !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + if(type != MJB_CASE_UPPER && type != MJB_CASE_LOWER && type != MJB_CASE_TITLE && type != MJB_CASE_CASEFOLD && type != MJB_CASE_CASEFOLD_SIMPLE) { return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -744,11 +760,39 @@ MJB_EXPORT mjb_status mjb_map_case(const char *buffer, size_t byte_length, mjb_e return MJB_STATUS_OK; } + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); + + if(status != MJB_STATUS_OK) { + return status; + } + bool turkic_case_folding = (type == MJB_CASE_CASEFOLD || type == MJB_CASE_CASEFOLD_SIMPLE) && (mjb_global.locale == MJB_LOCALE_TR || mjb_global.locale == MJB_LOCALE_AZ); - return mjb_map_case_result(buffer, byte_length, encoding, type, output_encoding, + status = mjb_map_case_result(buffer, byte_length, encoding, type, output_encoding, turkic_case_folding, result); + + mjb_result sanitized = { NULL, 0, false }; + + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); + + if(status == MJB_STATUS_OK && byte_length == 0) { + *result = sanitized; + + return MJB_STATUS_OK; + } + + if(status == MJB_STATUS_OK) { + status = mjb_map_case_result(buffer, byte_length, encoding, type, output_encoding, + turkic_case_folding, result); + } + } + + mjb_result_free(&sanitized); + + return status; } mjb_status mjb_casefold_default(const char *buffer, size_t byte_length, mjb_encoding encoding, @@ -757,6 +801,10 @@ mjb_status mjb_casefold_default(const char *buffer, size_t byte_length, mjb_enco return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -771,18 +819,26 @@ mjb_status mjb_casefold_default(const char *buffer, size_t byte_length, mjb_enco return MJB_STATUS_OK; } + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); + + if(status != MJB_STATUS_OK) { + return status; + } + return mjb_map_case_result(buffer, byte_length, encoding, MJB_CASE_CASEFOLD, output_encoding, false, result); } MJB_EXPORT mjb_status mjb_map_case_into(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_map_case_type type, mjb_encoding output_encoding, void *output, - size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_map_case_type type, + mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic) { if(output_size == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } - if(buffer == NULL && byte_length > 0) { + mjb_diagnostic_reset(diagnostic); + + if((buffer == NULL && byte_length > 0) || !mjb_malformed_policy_is_valid(malformed_policy)) { *output_size = 0; return MJB_STATUS_INVALID_ARGUMENT; @@ -795,6 +851,12 @@ MJB_EXPORT mjb_status mjb_map_case_into(const char *buffer, size_t byte_length, return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -803,10 +865,38 @@ MJB_EXPORT mjb_status mjb_map_case_into(const char *buffer, size_t byte_length, return status; } + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); + + if(status != MJB_STATUS_OK) { + *output_size = 0; + + return status; + } + bool turkic_case_folding = (type == MJB_CASE_CASEFOLD || type == MJB_CASE_CASEFOLD_SIMPLE) && (mjb_global.locale == MJB_LOCALE_TR || mjb_global.locale == MJB_LOCALE_AZ); mjb_map_case_write_context context = { buffer, byte_length, encoding, type, output_encoding, turkic_case_folding }; - return mjb_output_into(output, output_size, mjb_map_case_write, &context); + size_t output_capacity = *output_size; + status = mjb_output_into(output, output_size, mjb_map_case_write, &context); + + mjb_result sanitized = { NULL, 0, false }; + + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); + + if(status == MJB_STATUS_OK) { + *output_size = output_capacity; + context.buffer = buffer; + context.byte_length = byte_length; + context.encoding = encoding; + status = mjb_output_into(output, output_size, mjb_map_case_write, &context); + } + } + + mjb_result_free(&sanitized); + + return status; } diff --git a/src/caseless.c b/src/caseless.c index 2a2d6b93..3eeee7d1 100644 --- a/src/caseless.c +++ b/src/caseless.c @@ -47,8 +47,8 @@ static void mjb_caseless_value_replace(mjb_caseless_value *value, const mjb_resu static mjb_status mjb_caseless_normalize(mjb_caseless_value *value, mjb_normalization form) { mjb_result next; - mjb_status status = mjb_normalize(value->buffer, value->byte_length, value->encoding, form, - MJB_ENC_UTF_8, &next); + mjb_status status = mjb_normalize(value->buffer, value->byte_length, value->encoding, + MJB_MALFORMED_STOP, form, MJB_ENC_UTF_8, &next, NULL); if(status == MJB_STATUS_OK) { mjb_caseless_value_replace(value, &next); @@ -72,7 +72,7 @@ static mjb_status mjb_caseless_casefold(mjb_caseless_value *value) { static mjb_status mjb_caseless_nfkc_casefold(mjb_caseless_value *value) { mjb_result next; mjb_status status = mjb_nfkc_casefold(value->buffer, value->byte_length, value->encoding, - MJB_ENC_UTF_8, &next); + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &next, NULL); if(status == MJB_STATUS_OK) { mjb_caseless_value_replace(value, &next); @@ -156,27 +156,27 @@ MJB_EXPORT mjb_status mjb_caseless_match(const char *s1, size_t s1_byte_length, return status; } - // Second string. - status = mjb_resolve_input_byte_length(s2, &s2_byte_length, s2_encoding); + status = s1_byte_length == 0 ? + MJB_STATUS_OK : + mjb_check_input_encoding_byte_order(s1, s1_byte_length, s1_encoding); if(status != MJB_STATUS_OK) { return status; } - if(s1_byte_length > 0) { - status = mjb_validate_code_unit_sequence(s1, s1_byte_length, s1_encoding); + // Second string. + status = mjb_resolve_input_byte_length(s2, &s2_byte_length, s2_encoding); - if(status != MJB_STATUS_OK) { - return status; - } + if(status != MJB_STATUS_OK) { + return status; } - if(s2_byte_length > 0) { - status = mjb_validate_code_unit_sequence(s2, s2_byte_length, s2_encoding); + status = s2_byte_length == 0 ? + MJB_STATUS_OK : + mjb_check_input_encoding_byte_order(s2, s2_byte_length, s2_encoding); - if(status != MJB_STATUS_OK) { - return status; - } + if(status != MJB_STATUS_OK) { + return status; } mjb_caseless_value left = { s1, s1_byte_length, s1_encoding, false }; diff --git a/src/collation.c b/src/collation.c index 7e93da44..b3118fd4 100644 --- a/src/collation.c +++ b/src/collation.c @@ -806,8 +806,8 @@ static bool mjb_collation_strength_is_valid(mjb_collation_strength strength) { } static mjb_status compute_sort_key(const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, - mjb_sort_key *sk) { + mjb_malformed_policy malformed_policy, mjb_collation_variable_weighting variable_weighting, + mjb_collation_strength strength, mjb_sort_key *sk, mjb_diagnostic *diagnostic) { sk->data = NULL; sk->count = 0; sk->cap = 0; @@ -817,6 +817,10 @@ static mjb_status compute_sort_key(const char *buffer, size_t byte_length, mjb_e return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -827,14 +831,9 @@ static mjb_status compute_sort_key(const char *buffer, size_t byte_length, mjb_e return MJB_STATUS_OK; } - status = mjb_validate_code_unit_sequence(buffer, byte_length, encoding); - - if(status != MJB_STATUS_OK) { - return status; - } - mjb_result r; - status = mjb_normalize(buffer, byte_length, encoding, MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, &r); + status = mjb_normalize(buffer, byte_length, encoding, malformed_policy, MJB_NORMALIZATION_NFD, + MJB_ENC_UTF_8, &r, diagnostic); if(status != MJB_STATUS_OK) { return status; @@ -937,20 +936,24 @@ static mjb_status mjb_collation_key_byte_count(const mjb_sort_key *sort_key, siz } MJB_EXPORT mjb_status mjb_collation_key(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_collation_variable_weighting variable_weighting, - mjb_collation_strength strength, mjb_result *result) { - if(result == NULL || (buffer == NULL && byte_length > 0)) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, + mjb_result *result, mjb_diagnostic *diagnostic) { + if(result == NULL || (buffer == NULL && byte_length > 0) || + !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + result->output = NULL; result->output_size = 0; result->transformed = false; mjb_sort_key sk = { 0, 0, 0 }; - mjb_status status = compute_sort_key(buffer, byte_length, encoding, variable_weighting, - strength, &sk); + mjb_status status = compute_sort_key(buffer, byte_length, encoding, malformed_policy, + variable_weighting, strength, &sk, diagnostic); if(status != MJB_STATUS_OK) { return status; @@ -998,21 +1001,24 @@ MJB_EXPORT mjb_status mjb_collation_key(const char *buffer, size_t byte_length, } MJB_EXPORT mjb_status mjb_collation_key_into(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_collation_variable_weighting variable_weighting, - mjb_collation_strength strength, void *output, size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, + void *output, size_t *output_size, mjb_diagnostic *diagnostic) { if(output_size == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } - if(buffer == NULL && byte_length > 0) { + mjb_diagnostic_reset(diagnostic); + + if((buffer == NULL && byte_length > 0) || !mjb_malformed_policy_is_valid(malformed_policy)) { *output_size = 0; return MJB_STATUS_INVALID_ARGUMENT; } mjb_sort_key sort_key = { 0, 0, 0 }; - mjb_status status = compute_sort_key(buffer, byte_length, encoding, variable_weighting, - strength, &sort_key); + mjb_status status = compute_sort_key(buffer, byte_length, encoding, malformed_policy, + variable_weighting, strength, &sort_key, diagnostic); if(status != MJB_STATUS_OK) { *output_size = 0; @@ -1069,13 +1075,15 @@ MJB_EXPORT mjb_status mjb_collation_compare(const char *s1, size_t s1_byte_lengt mjb_sort_key sk1 = { 0, 0, 0 }; mjb_sort_key sk2 = { 0, 0, 0 }; - status = compute_sort_key(s1, s1_byte_length, s1_encoding, variable_weighting, strength, &sk1); + status = compute_sort_key(s1, s1_byte_length, s1_encoding, MJB_MALFORMED_STOP, + variable_weighting, strength, &sk1, NULL); if(status != MJB_STATUS_OK) { return status; } - status = compute_sort_key(s2, s2_byte_length, s2_encoding, variable_weighting, strength, &sk2); + status = compute_sort_key(s2, s2_byte_length, s2_encoding, MJB_MALFORMED_STOP, + variable_weighting, strength, &sk2, NULL); if(status != MJB_STATUS_OK) { sk_free(&sk1); @@ -1094,27 +1102,33 @@ MJB_EXPORT mjb_status mjb_collation_compare(const char *s1, size_t s1_byte_lengt #else MJB_EXPORT mjb_status mjb_collation_key(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_collation_variable_weighting variable_weighting, - mjb_collation_strength strength, mjb_result *result) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, + mjb_result *result, mjb_diagnostic *diagnostic) { (void)buffer; (void)byte_length; (void)encoding; + (void)malformed_policy; (void)variable_weighting; (void)strength; (void)result; + (void)diagnostic; return MJB_STATUS_FEATURE_NOT_ENABLED; } MJB_EXPORT mjb_status mjb_collation_key_into(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_collation_variable_weighting variable_weighting, - mjb_collation_strength strength, void *output, size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, + void *output, size_t *output_size, mjb_diagnostic *diagnostic) { (void)buffer; (void)byte_length; (void)encoding; + (void)malformed_policy; (void)variable_weighting; (void)strength; (void)output; (void)output_size; + (void)diagnostic; return MJB_STATUS_FEATURE_NOT_ENABLED; } diff --git a/src/cpp/mojibake.hpp b/src/cpp/mojibake.hpp index 6ba4149d..010e75b7 100644 --- a/src/cpp/mojibake.hpp +++ b/src/cpp/mojibake.hpp @@ -683,32 +683,65 @@ struct NumericValue { return mjb_is_utf16(input.data(), input.size()); } +[[nodiscard]] inline mjb_status validate_string(std::string_view input, + mjb_encoding encoding = MJB_ENC_UTF_8, mjb_diagnostic *diagnostic = nullptr) noexcept { + return mjb_string_validate(input.data(), input.size(), encoding, diagnostic); +} + +[[nodiscard]] inline mjb_status decode_next(std::string_view input, size_t &offset, + mjb_codepoint &codepoint, mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) noexcept { + return mjb_decode_next(input.data(), input.size(), encoding, malformed_policy, &offset, + &codepoint, diagnostic); +} + +[[nodiscard]] inline mjb_status decode_previous(std::string_view input, size_t &offset, + mjb_codepoint &codepoint, mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) noexcept { + return mjb_decode_previous(input.data(), input.size(), encoding, malformed_policy, &offset, + &codepoint, diagnostic); +} + [[nodiscard]] inline size_t codepoint_count(std::string_view input, - mjb_encoding encoding = MJB_ENC_UTF_8) { + mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { size_t count = 0; - detail::check_status(mjb_codepoint_count(input.data(), input.size(), encoding, &count), + detail::check_status(mjb_codepoint_count(input.data(), input.size(), encoding, malformed_policy, + &count, diagnostic), "Codepoint count failed"); return count; } [[nodiscard]] inline mjb_status for_each_codepoint(std::string_view input, - mjb_for_each_codepoint_fn callback, mjb_encoding encoding = MJB_ENC_UTF_8) noexcept { - return mjb_for_each_codepoint(input.data(), input.size(), encoding, callback); + mjb_for_each_codepoint_fn callback, mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) noexcept { + return mjb_for_each_codepoint(input.data(), input.size(), encoding, malformed_policy, callback, + diagnostic); } [[nodiscard]] inline TextResult convert_encoding_result(std::string_view input, - mjb_encoding input_encoding, mjb_encoding output_encoding) { + mjb_encoding input_encoding, mjb_encoding output_encoding, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { TextResult result = detail::ResultAccess::create(); const mjb_status status = mjb_convert_encoding(input.data(), input.size(), input_encoding, - output_encoding, detail::ResultAccess::out(result)); + malformed_policy, output_encoding, detail::ResultAccess::out(result), diagnostic); return detail::ResultAccess::checked(std::move(result), status, "Encoding conversion failed"); } [[nodiscard]] inline std::string convert_encoding(std::string_view input, - mjb_encoding input_encoding, mjb_encoding output_encoding) { - return convert_encoding_result(input, input_encoding, output_encoding).str(); + mjb_encoding input_encoding, mjb_encoding output_encoding, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return convert_encoding_result(input, input_encoding, output_encoding, malformed_policy, + diagnostic) + .str(); } [[nodiscard]] inline bool is_identifier(std::string_view input, @@ -750,37 +783,56 @@ enum class NormalizationForm { }; [[nodiscard]] inline TextResult normalize_result(std::string_view input, NormalizationForm form, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { TextResult result = detail::ResultAccess::create(); const mjb_status status = mjb_normalize(input.data(), input.size(), input_encoding, - static_cast(form), output_encoding, detail::ResultAccess::out(result)); + malformed_policy, static_cast(form), output_encoding, + detail::ResultAccess::out(result), diagnostic); return detail::ResultAccess::checked(std::move(result), status, "Normalization failed"); } [[nodiscard]] inline std::string normalize(std::string_view input, NormalizationForm form, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return normalize_result(input, form, input_encoding, output_encoding).str(); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return normalize_result(input, form, input_encoding, output_encoding, malformed_policy, + diagnostic) + .str(); } [[nodiscard]] inline std::string nfc(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return normalize(input, NormalizationForm::NFC, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return normalize(input, NormalizationForm::NFC, input_encoding, output_encoding, + malformed_policy, diagnostic); } [[nodiscard]] inline std::string nfd(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return normalize(input, NormalizationForm::NFD, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return normalize(input, NormalizationForm::NFD, input_encoding, output_encoding, + malformed_policy, diagnostic); } [[nodiscard]] inline std::string nfkc(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return normalize(input, NormalizationForm::NFKC, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return normalize(input, NormalizationForm::NFKC, input_encoding, output_encoding, + malformed_policy, diagnostic); } [[nodiscard]] inline std::string nfkd(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return normalize(input, NormalizationForm::NFKD, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return normalize(input, NormalizationForm::NFKD, input_encoding, output_encoding, + malformed_policy, diagnostic); } [[nodiscard]] inline mjb_quick_check_result normalization_quick_check(std::string_view input, @@ -818,41 +870,59 @@ constexpr Filter &operator|=(Filter &left, Filter right) noexcept { } [[nodiscard]] inline TextResult filter_result(std::string_view input, mjb_filter_flags filters, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { TextResult result = detail::ResultAccess::create(); - const mjb_status status = mjb_filter(input.data(), input.size(), input_encoding, filters, - output_encoding, detail::ResultAccess::out(result)); + const mjb_status status = mjb_filter(input.data(), input.size(), input_encoding, + malformed_policy, filters, output_encoding, detail::ResultAccess::out(result), diagnostic); return detail::ResultAccess::checked(std::move(result), status, "String filtering failed"); } [[nodiscard]] inline std::string filter(std::string_view input, mjb_filter_flags filters, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return filter_result(input, filters, input_encoding, output_encoding).str(); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return filter_result(input, filters, input_encoding, output_encoding, malformed_policy, + diagnostic) + .str(); } [[nodiscard]] inline TextResult filter_result(std::string_view input, Filter filters, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { return filter_result(input, static_cast(filters), input_encoding, - output_encoding); + output_encoding, malformed_policy, diagnostic); } [[nodiscard]] inline std::string filter(std::string_view input, Filter filters, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return filter_result(input, filters, input_encoding, output_encoding).str(); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return filter_result(input, filters, input_encoding, output_encoding, malformed_policy, + diagnostic) + .str(); } [[nodiscard]] inline TextResult nfkc_casefold_result(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { TextResult result = detail::ResultAccess::create(); const mjb_status status = mjb_nfkc_casefold(input.data(), input.size(), input_encoding, - output_encoding, detail::ResultAccess::out(result)); + malformed_policy, output_encoding, detail::ResultAccess::out(result), diagnostic); return detail::ResultAccess::checked(std::move(result), status, "NFKC case folding failed"); } [[nodiscard]] inline std::string nfkc_casefold(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return nfkc_casefold_result(input, input_encoding, output_encoding).str(); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return nfkc_casefold_result(input, input_encoding, output_encoding, malformed_policy, + diagnostic) + .str(); } class IdnaResult { @@ -912,42 +982,63 @@ class IdnaResult { } [[nodiscard]] inline TextResult case_map_result(std::string_view input, mjb_map_case_type type, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { TextResult result = detail::ResultAccess::create(); - const mjb_status status = mjb_map_case(input.data(), input.size(), input_encoding, type, - output_encoding, detail::ResultAccess::out(result)); + const mjb_status status = mjb_map_case(input.data(), input.size(), input_encoding, + malformed_policy, type, output_encoding, detail::ResultAccess::out(result), diagnostic); return detail::ResultAccess::checked(std::move(result), status, "Case mapping failed"); } [[nodiscard]] inline std::string case_map(std::string_view input, mjb_map_case_type type, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return case_map_result(input, type, input_encoding, output_encoding).str(); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return case_map_result(input, type, input_encoding, output_encoding, malformed_policy, + diagnostic) + .str(); } [[nodiscard]] inline std::string uppercase(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return case_map(input, MJB_CASE_UPPER, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return case_map(input, MJB_CASE_UPPER, input_encoding, output_encoding, malformed_policy, + diagnostic); } [[nodiscard]] inline std::string lowercase(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return case_map(input, MJB_CASE_LOWER, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return case_map(input, MJB_CASE_LOWER, input_encoding, output_encoding, malformed_policy, + diagnostic); } [[nodiscard]] inline std::string titlecase(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return case_map(input, MJB_CASE_TITLE, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return case_map(input, MJB_CASE_TITLE, input_encoding, output_encoding, malformed_policy, + diagnostic); } [[nodiscard]] inline std::string casefold(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return case_map(input, MJB_CASE_CASEFOLD, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return case_map(input, MJB_CASE_CASEFOLD, input_encoding, output_encoding, malformed_policy, + diagnostic); } [[nodiscard]] inline std::string casefold_simple(std::string_view input, - mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8) { - return case_map(input, MJB_CASE_CASEFOLD_SIMPLE, input_encoding, output_encoding); + mjb_encoding input_encoding = MJB_ENC_UTF_8, mjb_encoding output_encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return case_map(input, MJB_CASE_CASEFOLD_SIMPLE, input_encoding, output_encoding, + malformed_policy, diagnostic); } enum class CaselessMode { @@ -996,12 +1087,14 @@ enum class CollationVariableWeighting { [[nodiscard]] inline TextResult collation_key_result(std::string_view input, CollationVariableWeighting variable_weighting = CollationVariableWeighting::NonIgnorable, - CollationStrength strength = CollationStrength::Tertiary, - mjb_encoding encoding = MJB_ENC_UTF_8) { + CollationStrength strength = CollationStrength::Tertiary, mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { TextResult result = detail::ResultAccess::create(); const mjb_status status = mjb_collation_key(input.data(), input.size(), encoding, - static_cast(variable_weighting), - static_cast(strength), detail::ResultAccess::out(result)); + malformed_policy, static_cast(variable_weighting), + static_cast(strength), detail::ResultAccess::out(result), + diagnostic); return detail::ResultAccess::checked(std::move(result), status, "Collation key generation failed"); @@ -1009,9 +1102,12 @@ enum class CollationVariableWeighting { [[nodiscard]] inline std::string collation_key(std::string_view input, CollationVariableWeighting variable_weighting = CollationVariableWeighting::NonIgnorable, - CollationStrength strength = CollationStrength::Tertiary, - mjb_encoding encoding = MJB_ENC_UTF_8) { - return collation_key_result(input, variable_weighting, strength, encoding).str(); + CollationStrength strength = CollationStrength::Tertiary, mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { + return collation_key_result(input, variable_weighting, strength, encoding, malformed_policy, + diagnostic) + .str(); } struct EmojiSequence { @@ -1057,9 +1153,12 @@ struct EmojiSequence { [[nodiscard]] inline size_t terminal_width(std::string_view input, mjb_terminal_width_profile profile = MJB_TERMINAL_WIDTH_NARROW, - mjb_encoding encoding = MJB_ENC_UTF_8) { + mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { size_t width = 0; - detail::check_status(mjb_terminal_width(input.data(), input.size(), encoding, profile, &width), + detail::check_status(mjb_terminal_width(input.data(), input.size(), encoding, malformed_policy, + profile, &width, diagnostic), "Terminal width calculation failed"); return width; @@ -1174,27 +1273,36 @@ inline void set_allocator(const mjb_allocator *allocator) { } [[nodiscard]] inline size_t grapheme_count(std::string_view input, - mjb_encoding encoding = MJB_ENC_UTF_8) { + mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { size_t count = 0; - detail::check_status(mjb_grapheme_count(input.data(), input.size(), encoding, &count), + detail::check_status(mjb_grapheme_count(input.data(), input.size(), encoding, malformed_policy, + &count, diagnostic), "Grapheme count failed"); return count; } [[nodiscard]] inline size_t sentence_count(std::string_view input, - mjb_encoding encoding = MJB_ENC_UTF_8) { + mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { size_t count = 0; - detail::check_status(mjb_sentence_count(input.data(), input.size(), encoding, &count), + detail::check_status(mjb_sentence_count(input.data(), input.size(), encoding, malformed_policy, + &count, diagnostic), "Sentence count failed"); return count; } [[nodiscard]] inline size_t word_count(std::string_view input, - mjb_encoding encoding = MJB_ENC_UTF_8) { + mjb_encoding encoding = MJB_ENC_UTF_8, + mjb_malformed_policy malformed_policy = MJB_MALFORMED_STOP, + mjb_diagnostic *diagnostic = nullptr) { size_t count = 0; - detail::check_status(mjb_word_count(input.data(), input.size(), encoding, &count), + detail::check_status(mjb_word_count(input.data(), input.size(), encoding, malformed_policy, + &count, diagnostic), "Word count failed"); return count; diff --git a/src/encoding.c b/src/encoding.c index 1a910538..eeecad69 100644 --- a/src/encoding.c +++ b/src/encoding.c @@ -14,7 +14,9 @@ typedef struct mjb_convert_encoding_context { size_t byte_length; size_t input_index; mjb_encoding input_encoding; + mjb_malformed_policy malformed_policy; mjb_encoding output_encoding; + mjb_diagnostic *diagnostic; } mjb_convert_encoding_context; static bool mjb_codepoint_is_surrogate(mjb_codepoint codepoint) { @@ -84,100 +86,611 @@ MJB_EXPORT mjb_encoding mjb_detect_encoding(const char *buffer, size_t byte_leng * Return true if the string is encoded in UTF-8. */ MJB_EXPORT bool mjb_is_utf8(const char *buffer, size_t byte_length) { - if(buffer == NULL || byte_length == 0) { - return false; + return mjb_string_validate(buffer, byte_length, MJB_ENC_UTF_8, NULL) == MJB_STATUS_OK; +} + +/** + * Return true if the string is encoded in ASCII. + */ +MJB_EXPORT bool mjb_is_ascii(const char *buffer, size_t byte_length) { + return mjb_string_validate(buffer, byte_length, MJB_ENC_ASCII, NULL) == MJB_STATUS_OK; +} + +/** + * Return true if the string is encoded in UTF-16BE or UTF-16LE. + */ +MJB_EXPORT bool mjb_is_utf16(const char *buffer, size_t byte_length) { + return mjb_string_validate(buffer, byte_length, MJB_ENC_UTF_16BE, NULL) == MJB_STATUS_OK || + mjb_string_validate(buffer, byte_length, MJB_ENC_UTF_16LE, NULL) == MJB_STATUS_OK; +} + +static size_t mjb_encoding_code_unit_size(mjb_encoding encoding) { + if(encoding == MJB_ENC_UTF_16BE || encoding == MJB_ENC_UTF_16LE) { + return 2; + } + + if(encoding == MJB_ENC_UTF_32BE || encoding == MJB_ENC_UTF_32LE) { + return 4; + } + + return 1; +} + +static void mjb_set_diagnostic(mjb_diagnostic *diagnostic, mjb_text_error error, size_t byte_offset, + size_t byte_length, mjb_encoding encoding) { + if(diagnostic == NULL) { + return; + } + + diagnostic->error = error; + diagnostic->byte_offset = byte_offset; + diagnostic->byte_length = byte_length; + diagnostic->code_unit_offset = byte_offset / mjb_encoding_code_unit_size(encoding); +} + +static uint16_t mjb_read_utf16_unit(const char *buffer, size_t offset, bool big_endian) { + uint16_t first = (uint8_t)buffer[offset]; + uint16_t second = (uint8_t)buffer[offset + 1]; + + return big_endian ? (uint16_t)((first << 8) | second) : (uint16_t)(first | (second << 8)); +} + +static uint32_t mjb_read_utf32_unit(const char *buffer, size_t offset, bool big_endian) { + uint32_t b0 = (uint8_t)buffer[offset]; + uint32_t b1 = (uint8_t)buffer[offset + 1]; + uint32_t b2 = (uint8_t)buffer[offset + 2]; + uint32_t b3 = (uint8_t)buffer[offset + 3]; + + if(big_endian) { + return (b0 << 24) | (b1 << 16) | (b2 << 8) | b3; + } + + return b0 | (b1 << 8) | (b2 << 16) | (b3 << 24); +} + +static mjb_status mjb_decode_next_raw(const char *buffer, size_t byte_length, mjb_encoding encoding, + size_t *offset, mjb_codepoint *codepoint, mjb_diagnostic *diagnostic) { + size_t start = *offset; + + if(start >= byte_length) { + return MJB_STATUS_END_OF_INPUT; + } + + if(encoding == MJB_ENC_ASCII) { + uint8_t byte = (uint8_t)buffer[start]; + *offset = start + 1; + + if(byte <= 0x7F) { + *codepoint = byte; + + return MJB_STATUS_OK; + } + + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_NON_ASCII, start, 1, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + if(encoding == MJB_ENC_UTF_8) { + uint8_t first = (uint8_t)buffer[start]; + + if(first <= 0x7F) { + *offset = start + 1; + *codepoint = first; + + return MJB_STATUS_OK; + } + + if(first >= 0x80 && first <= 0xBF) { + *offset = start + 1; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, start, 1, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + if(first == 0xC0 || first == 0xC1) { + *offset = start + 1; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_OVERLONG_SEQUENCE, start, 1, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + size_t required; + + if(first <= 0xDF) { + required = 2; + } else if(first <= 0xEF) { + required = 3; + } else if(first <= 0xF4) { + required = 4; + } else { + *offset = start + 1; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, + first <= 0xF7 ? MJB_TEXT_ERROR_OUT_OF_RANGE : MJB_TEXT_ERROR_INVALID_LEADING_BYTE, + start, 1, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + if(byte_length - start < 2) { + *offset = byte_length; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, start, + byte_length - start, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + uint8_t second = (uint8_t)buffer[start + 1]; + + if((second & 0xC0) != 0x80) { + *offset = start + 1; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_MISSING_CONTINUATION, start, 1, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + mjb_text_error constrained_error = MJB_TEXT_ERROR_NONE; + + if((first == 0xE0 && second < 0xA0) || (first == 0xF0 && second < 0x90)) { + constrained_error = MJB_TEXT_ERROR_OVERLONG_SEQUENCE; + } else if(first == 0xED && second >= 0xA0) { + constrained_error = MJB_TEXT_ERROR_SURROGATE; + } else if(first == 0xF4 && second >= 0x90) { + constrained_error = MJB_TEXT_ERROR_OUT_OF_RANGE; + } + + if(constrained_error != MJB_TEXT_ERROR_NONE) { + // Only the lead byte is a maximal subpart when the second byte violates the + // well-formed UTF-8 range for that lead byte. + *offset = start + 1; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, constrained_error, start, 1, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + for(size_t i = 2; i < required; ++i) { + if(start + i >= byte_length) { + *offset = byte_length; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, start, + byte_length - start, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + if(((uint8_t)buffer[start + i] & 0xC0) != 0x80) { + *offset = start + i; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_MISSING_CONTINUATION, start, i, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + } + + if(required == 2) { + *codepoint = ((mjb_codepoint)(first & 0x1F) << 6) | ((mjb_codepoint)second & 0x3F); + } else if(required == 3) { + *codepoint = ((mjb_codepoint)(first & 0x0F) << 12) | + (((mjb_codepoint)second & 0x3F) << 6) | + ((mjb_codepoint)(uint8_t)buffer[start + 2] & 0x3F); + } else { + *codepoint = ((mjb_codepoint)(first & 0x07) << 18) | + (((mjb_codepoint)second & 0x3F) << 12) | + (((mjb_codepoint)(uint8_t)buffer[start + 2] & 0x3F) << 6) | + ((mjb_codepoint)(uint8_t)buffer[start + 3] & 0x3F); + } + + *offset = start + required; + + return MJB_STATUS_OK; } - if(mjb_resolve_input_byte_length(buffer, &byte_length, MJB_ENC_UTF_8) != MJB_STATUS_OK || - byte_length == 0) { - return false; + if(encoding == MJB_ENC_UTF_16BE || encoding == MJB_ENC_UTF_16LE) { + size_t remaining = byte_length - start; + + if(remaining < 2) { + *offset = byte_length; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_CODE_UNIT, start, remaining, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + bool big_endian = encoding == MJB_ENC_UTF_16BE; + uint16_t first = mjb_read_utf16_unit(buffer, start, big_endian); + + if(first >= 0xD800 && first <= 0xDBFF) { + if(remaining < 4) { + *offset = start + 2; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_UNPAIRED_SURROGATE, start, 2, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + uint16_t second = mjb_read_utf16_unit(buffer, start + 2, big_endian); + + if(second < 0xDC00 || second > 0xDFFF) { + *offset = start + 2; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_UNPAIRED_SURROGATE, start, 2, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + *codepoint = 0x10000 + (((mjb_codepoint)first - 0xD800) << 10) + + ((mjb_codepoint)second - 0xDC00); + *offset = start + 4; + + return MJB_STATUS_OK; + } + + *offset = start + 2; + + if(first >= 0xDC00 && first <= 0xDFFF) { + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_UNPAIRED_SURROGATE, start, 2, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + *codepoint = first; + + return MJB_STATUS_OK; } - uint8_t state = MJB_UTF_ACCEPT; - mjb_codepoint codepoint = MJB_CODEPOINT_NOT_VALID; + if(encoding == MJB_ENC_UTF_32BE || encoding == MJB_ENC_UTF_32LE) { + size_t remaining = byte_length - start; - // Loop through the string. - for(size_t i = 0; i < byte_length; ++i) { - // Find next codepoint. - state = mjb_utf8_decode_step(state, buffer[i], &codepoint); + if(remaining < 4) { + *offset = byte_length; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_CODE_UNIT, start, remaining, + encoding); - if(state == MJB_UTF_REJECT) { - // The string is not well-formed. - return false; + return MJB_STATUS_MALFORMED_INPUT; } + + mjb_codepoint decoded = mjb_read_utf32_unit(buffer, start, encoding == MJB_ENC_UTF_32BE); + *offset = start + 4; + + if(mjb_codepoint_is_surrogate(decoded)) { + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_SURROGATE, start, 4, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + if(decoded > MJB_CODEPOINT_MAX) { + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_OUT_OF_RANGE, start, 4, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + *codepoint = decoded; + + return MJB_STATUS_OK; } - return state == MJB_UTF_ACCEPT; + return MJB_STATUS_INVALID_ENCODING; } -/** - * Return true if the string is encoded in ASCII. - */ -MJB_EXPORT bool mjb_is_ascii(const char *buffer, size_t byte_length) { - if(buffer == NULL || byte_length == 0) { - return false; +static mjb_status mjb_resolve_decode_input(const char *buffer, size_t *byte_length, + mjb_encoding requested_encoding, size_t *offset, mjb_encoding *resolved_encoding) { + if(!mjb_encoding_is_valid_input(requested_encoding)) { + return MJB_STATUS_INVALID_ENCODING; } - if(mjb_resolve_input_byte_length(buffer, &byte_length, MJB_ENC_ASCII) != MJB_STATUS_OK || - byte_length == 0) { - return false; + mjb_status status = mjb_resolve_input_byte_length(buffer, byte_length, requested_encoding); + + if(status != MJB_STATUS_OK) { + return status; } - for(size_t i = 0; i < byte_length; ++i) { - // Every character must have leading bit at zero. - if(buffer[i] & 0x80) { - return false; + if((buffer == NULL && *byte_length > 0) || *offset > *byte_length) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + size_t data_start = 0; + *resolved_encoding = mjb_resolve_input_encoding(buffer, *byte_length, requested_encoding, + &data_start); + + if((requested_encoding == MJB_ENC_UTF_16 || requested_encoding == MJB_ENC_UTF_32) && + *resolved_encoding == requested_encoding) { + return MJB_STATUS_INVALID_ENCODING; + } + + if(*offset < data_start) { + if(*offset != 0) { + return MJB_STATUS_INVALID_ARGUMENT; } + + *offset = data_start; } - return 1; + return MJB_STATUS_OK; } -/** - * Return true if the string is encoded in UTF-16BE or UTF-16LE. - */ -MJB_EXPORT bool mjb_is_utf16(const char *buffer, size_t byte_length) { - if(buffer == NULL || byte_length == 0) { - return false; +MJB_EXPORT mjb_status mjb_decode_next(const char *buffer, size_t byte_length, mjb_encoding encoding, + mjb_malformed_policy malformed_policy, size_t *offset, mjb_codepoint *codepoint, + mjb_diagnostic *diagnostic) { + if(offset == NULL || codepoint == NULL) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + mjb_diagnostic_reset(diagnostic); + *codepoint = MJB_CODEPOINT_NOT_VALID; + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; } - if(mjb_resolve_input_byte_length(buffer, &byte_length, MJB_ENC_UTF_16LE) != MJB_STATUS_OK || - byte_length < 2 || (byte_length % 2) != 0) { - return false; + mjb_encoding resolved_encoding; + mjb_status status = mjb_resolve_decode_input(buffer, &byte_length, encoding, offset, + &resolved_encoding); + + if(status != MJB_STATUS_OK) { + return status; } - // Try UTF-16BE first - uint8_t state_be = MJB_UTF_ACCEPT; - mjb_codepoint codepoint = MJB_CODEPOINT_NOT_VALID; - bool be_valid = true; + for(;;) { + mjb_diagnostic current; + mjb_diagnostic_reset(¤t); + status = mjb_decode_next_raw(buffer, byte_length, resolved_encoding, offset, codepoint, + ¤t); + mjb_diagnostic_record(diagnostic, ¤t); - for(size_t i = 0; i < byte_length; i += 2) { - state_be = mjb_utf16_decode_step(state_be, buffer[i], buffer[i + 1], &codepoint, true); + if(status != MJB_STATUS_MALFORMED_INPUT) { + return status; + } - if(state_be == MJB_UTF_REJECT) { - be_valid = false; // Error in UTF-16BE - break; + if(malformed_policy == MJB_MALFORMED_STOP) { + return status; + } + + if(malformed_policy == MJB_MALFORMED_REPLACE) { + *codepoint = MJB_CODEPOINT_REPLACEMENT; + + return MJB_STATUS_OK; } } +} + +static mjb_status mjb_decode_previous_raw(const char *buffer, size_t data_start, + mjb_encoding encoding, size_t *offset, mjb_codepoint *codepoint, mjb_diagnostic *diagnostic) { + size_t end = *offset; + + if(end <= data_start) { + return MJB_STATUS_END_OF_INPUT; + } - if(be_valid && state_be == MJB_UTF_ACCEPT) { - return true; // Valid UTF-16BE + if(encoding == MJB_ENC_ASCII) { + size_t start = end - 1; + size_t next = start; + mjb_status status = mjb_decode_next_raw(buffer, end, encoding, &next, codepoint, + diagnostic); + *offset = start; + + return status; } - // Try UTF-16LE - uint8_t state_le = MJB_UTF_ACCEPT; - bool le_valid = true; + if(encoding == MJB_ENC_UTF_8) { + size_t candidate = end - 1; + size_t continuation_count = 0; - for(size_t i = 0; i < byte_length; i += 2) { - state_le = mjb_utf16_decode_step(state_le, buffer[i], buffer[i + 1], &codepoint, false); + while(candidate > data_start && (((uint8_t)buffer[candidate] & 0xC0) == 0x80) && + continuation_count < 3) { + --candidate; + ++continuation_count; + } - if(state_le == MJB_UTF_REJECT) { - le_valid = false; // Error in UTF-16LE - break; + size_t next = candidate; + mjb_status status = mjb_decode_next_raw(buffer, end, encoding, &next, codepoint, + diagnostic); + + if(next == end && (status == MJB_STATUS_OK || status == MJB_STATUS_MALFORMED_INPUT)) { + *offset = candidate; + + return status; + } + + candidate = end - 1; + *offset = candidate; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, + (((uint8_t)buffer[candidate] & 0xC0) == 0x80) ? MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION : + MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, + candidate, 1, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + if(encoding == MJB_ENC_UTF_16BE || encoding == MJB_ENC_UTF_16LE) { + size_t relative = end - data_start; + + if((relative % 2) != 0) { + *offset = end - 1; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_CODE_UNIT, end - 1, 1, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + bool big_endian = encoding == MJB_ENC_UTF_16BE; + size_t start = end - 2; + uint16_t last = mjb_read_utf16_unit(buffer, start, big_endian); + + if(last >= 0xDC00 && last <= 0xDFFF && start >= data_start + 2) { + uint16_t first = mjb_read_utf16_unit(buffer, start - 2, big_endian); + + if(first >= 0xD800 && first <= 0xDBFF) { + *offset = start - 2; + *codepoint = 0x10000 + (((mjb_codepoint)first - 0xD800) << 10) + + ((mjb_codepoint)last - 0xDC00); + + return MJB_STATUS_OK; + } + } + + *offset = start; + + if(last >= 0xD800 && last <= 0xDFFF) { + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_UNPAIRED_SURROGATE, start, 2, encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + *codepoint = last; + + return MJB_STATUS_OK; + } + + if(encoding == MJB_ENC_UTF_32BE || encoding == MJB_ENC_UTF_32LE) { + size_t relative = end - data_start; + size_t remainder = relative % 4; + + if(remainder != 0) { + size_t start = end - remainder; + *offset = start; + *codepoint = MJB_CODEPOINT_REPLACEMENT; + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_CODE_UNIT, start, remainder, + encoding); + + return MJB_STATUS_MALFORMED_INPUT; + } + + size_t start = end - 4; + size_t next = start; + mjb_status status = mjb_decode_next_raw(buffer, end, encoding, &next, codepoint, + diagnostic); + *offset = start; + + return status; + } + + return MJB_STATUS_INVALID_ENCODING; +} + +MJB_EXPORT mjb_status mjb_decode_previous(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *offset, + mjb_codepoint *codepoint, mjb_diagnostic *diagnostic) { + if(offset == NULL || codepoint == NULL) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + mjb_diagnostic_reset(diagnostic); + *codepoint = MJB_CODEPOINT_NOT_VALID; + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); + + if(status != MJB_STATUS_OK) { + return status; + } + + if((buffer == NULL && byte_length > 0) || *offset > byte_length) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + size_t data_start = 0; + mjb_encoding resolved_encoding = mjb_resolve_input_encoding(buffer, byte_length, encoding, + &data_start); + + if((encoding == MJB_ENC_UTF_16 || encoding == MJB_ENC_UTF_32) && + resolved_encoding == encoding) { + return MJB_STATUS_INVALID_ENCODING; + } + + if(*offset < data_start) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + for(;;) { + mjb_diagnostic current; + mjb_diagnostic_reset(¤t); + status = mjb_decode_previous_raw(buffer, data_start, resolved_encoding, offset, codepoint, + ¤t); + mjb_diagnostic_record(diagnostic, ¤t); + + if(status != MJB_STATUS_MALFORMED_INPUT) { + return status; + } + + if(malformed_policy == MJB_MALFORMED_STOP) { + return status; + } + + if(malformed_policy == MJB_MALFORMED_REPLACE) { + *codepoint = MJB_CODEPOINT_REPLACEMENT; + + return MJB_STATUS_OK; } } +} + +MJB_EXPORT mjb_status mjb_string_validate(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_diagnostic *diagnostic) { + mjb_diagnostic_reset(diagnostic); - return le_valid && state_le == MJB_UTF_ACCEPT; // Valid UTF-16LE + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); + + if(status != MJB_STATUS_OK) { + return status; + } + + if(buffer == NULL && byte_length > 0) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + size_t offset = 0; + mjb_encoding resolved_encoding = mjb_resolve_input_encoding(buffer, byte_length, encoding, + &offset); + + if((encoding == MJB_ENC_UTF_16 || encoding == MJB_ENC_UTF_32) && + resolved_encoding == encoding) { + return MJB_STATUS_INVALID_ENCODING; + } + + mjb_codepoint codepoint; + + for(;;) { + status = mjb_decode_next_raw(buffer, byte_length, resolved_encoding, &offset, &codepoint, + diagnostic); + + if(status == MJB_STATUS_END_OF_INPUT) { + return MJB_STATUS_OK; + } + + if(status != MJB_STATUS_OK) { + return status; + } + } } MJB_EXPORT unsigned int mjb_codepoint_encode(mjb_codepoint codepoint, char *buffer, @@ -320,20 +833,20 @@ MJB_EXPORT unsigned int mjb_codepoint_encode(mjb_codepoint codepoint, char *buff static mjb_status mjb_convert_encoding_write(mjb_output *output, const void *context_pointer) { const mjb_convert_encoding_context *context = (const mjb_convert_encoding_context *) context_pointer; - uint8_t state = MJB_UTF_ACCEPT; mjb_codepoint codepoint = 0; - bool in_error = false; - for(size_t i = context->input_index; i < context->byte_length;) { - mjb_decode_result decode_status = mjb_next_codepoint(context->buffer, context->byte_length, - &state, &i, context->input_encoding, &codepoint, &in_error); + for(size_t offset = context->input_index;;) { + mjb_diagnostic current; + mjb_status decode_status = mjb_decode_next(context->buffer, context->byte_length, + context->input_encoding, context->malformed_policy, &offset, &codepoint, ¤t); + mjb_diagnostic_record(context->diagnostic, ¤t); - if(decode_status == MJB_DECODE_END) { + if(decode_status == MJB_STATUS_END_OF_INPUT) { break; } - if(decode_status == MJB_DECODE_INCOMPLETE) { - continue; + if(decode_status != MJB_STATUS_OK) { + return decode_status; } mjb_status status = mjb_output_codepoint(output, codepoint, context->output_encoding); @@ -347,11 +860,19 @@ static mjb_status mjb_convert_encoding_write(mjb_output *output, const void *con } MJB_EXPORT mjb_status mjb_convert_encoding(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_encoding output_encoding, mjb_result *result) { - if(result == NULL || (buffer == NULL && byte_length > 0)) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, + mjb_result *result, mjb_diagnostic *diagnostic) { + if(result == NULL || (buffer == NULL && byte_length > 0) || + !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + + if(!mjb_encoding_is_valid_output(output_encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -362,22 +883,31 @@ MJB_EXPORT mjb_status mjb_convert_encoding(const char *buffer, size_t byte_lengt result->output_size = 0; result->transformed = false; - if(byte_length == 0 || encoding == output_encoding) { - result->output = (char *)buffer; - result->output_size = byte_length; - - return MJB_STATUS_OK; - } - size_t input_index = 0; mjb_encoding input_encoding = mjb_resolve_input_encoding(buffer, byte_length, encoding, &input_index); - if(input_encoding == MJB_ENC_UTF_16 || input_encoding == MJB_ENC_UTF_32 || - output_encoding == MJB_ENC_UTF_16 || output_encoding == MJB_ENC_UTF_32) { + if(input_encoding == MJB_ENC_UTF_16 || input_encoding == MJB_ENC_UTF_32) { return MJB_STATUS_INVALID_ENCODING; } + if(byte_length == 0 || encoding == output_encoding) { + mjb_diagnostic validity; + status = mjb_string_validate(buffer, byte_length, encoding, &validity); + mjb_diagnostic_record(diagnostic, &validity); + + if(status == MJB_STATUS_OK) { + result->output = (char *)buffer; + result->output_size = byte_length; + + return MJB_STATUS_OK; + } + + if(status != MJB_STATUS_MALFORMED_INPUT || malformed_policy == MJB_MALFORMED_STOP) { + return status; + } + } + char *allocated = (char *)mjb_alloc(byte_length); if(allocated == NULL) { @@ -387,7 +917,7 @@ MJB_EXPORT mjb_status mjb_convert_encoding(const char *buffer, size_t byte_lengt mjb_output output; mjb_output_init_dynamic(&output, allocated, byte_length); mjb_convert_encoding_context context = { buffer, byte_length, input_index, input_encoding, - output_encoding }; + malformed_policy, output_encoding, diagnostic }; status = mjb_convert_encoding_write(&output, &context); if(status != MJB_STATUS_OK) { @@ -404,17 +934,32 @@ MJB_EXPORT mjb_status mjb_convert_encoding(const char *buffer, size_t byte_lengt } MJB_EXPORT mjb_status mjb_convert_encoding_into(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_encoding output_encoding, void *output, size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, + void *output, size_t *output_size, mjb_diagnostic *diagnostic) { if(output_size == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ARGUMENT; + } + if(buffer == NULL && byte_length > 0) { *output_size = 0; return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_output(output_encoding)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -423,23 +968,34 @@ MJB_EXPORT mjb_status mjb_convert_encoding_into(const char *buffer, size_t byte_ return status; } - if(byte_length == 0 || encoding == output_encoding) { - return mjb_output_copy_into(buffer, byte_length, output, output_size); - } - size_t input_index = 0; mjb_encoding input_encoding = mjb_resolve_input_encoding(buffer, byte_length, encoding, &input_index); - if(input_encoding == MJB_ENC_UTF_16 || input_encoding == MJB_ENC_UTF_32 || - output_encoding == MJB_ENC_UTF_16 || output_encoding == MJB_ENC_UTF_32) { + if(input_encoding == MJB_ENC_UTF_16 || input_encoding == MJB_ENC_UTF_32) { *output_size = 0; return MJB_STATUS_INVALID_ENCODING; } + if(byte_length == 0 || encoding == output_encoding) { + mjb_diagnostic validity; + status = mjb_string_validate(buffer, byte_length, encoding, &validity); + mjb_diagnostic_record(diagnostic, &validity); + + if(status == MJB_STATUS_OK) { + return mjb_output_copy_into(buffer, byte_length, output, output_size); + } + + if(status != MJB_STATUS_MALFORMED_INPUT || malformed_policy == MJB_MALFORMED_STOP) { + *output_size = 0; + + return status; + } + } + mjb_convert_encoding_context context = { buffer, byte_length, input_index, input_encoding, - output_encoding }; + malformed_policy, output_encoding, diagnostic }; return mjb_output_into(output, output_size, mjb_convert_encoding_write, &context); } diff --git a/src/filter.c b/src/filter.c index afa1d653..a431a59d 100644 --- a/src/filter.c +++ b/src/filter.c @@ -11,34 +11,36 @@ typedef struct mjb_filter_context { const char *buffer; size_t byte_length; mjb_encoding encoding; + mjb_malformed_policy malformed_policy; mjb_filter_flags filters; mjb_encoding output_encoding; + mjb_diagnostic *diagnostic; } mjb_filter_context; static mjb_status mjb_filter_process(const mjb_filter_context *context, mjb_output *output, bool *transformed) { - uint8_t state = MJB_UTF_ACCEPT; mjb_codepoint codepoint = 0; mjb_character character; bool last_was_whitespace = false; size_t combining_mark_count = 0; bool any_transformation = false; - bool in_error = false; - for(size_t i = 0; i < context->byte_length;) { - mjb_decode_result decode_status = mjb_next_codepoint(context->buffer, context->byte_length, - &state, &i, context->encoding, &codepoint, &in_error); + for(size_t offset = 0;;) { + mjb_diagnostic current; + mjb_status decode_status = mjb_decode_next(context->buffer, context->byte_length, + context->encoding, context->malformed_policy, &offset, &codepoint, ¤t); - if(decode_status == MJB_DECODE_END) { - break; + if(current.error != MJB_TEXT_ERROR_NONE) { + any_transformation = true; + mjb_diagnostic_record(context->diagnostic, ¤t); } - if(decode_status == MJB_DECODE_INCOMPLETE) { - continue; + if(decode_status == MJB_STATUS_END_OF_INPUT) { + break; } - if(decode_status == MJB_DECODE_ERROR) { - any_transformation = true; + if(decode_status != MJB_STATUS_OK) { + return decode_status; } // Get current character. @@ -127,20 +129,6 @@ static mjb_status mjb_filter_process(const mjb_filter_context *context, mjb_outp last_was_whitespace = is_whitespace; } - if(mjb_utf_state_is_incomplete(state)) { - // Incomplete multibyte sequence at end of string - if(!in_error) { - mjb_status status = mjb_output_codepoint(output, MJB_CODEPOINT_REPLACEMENT, - context->output_encoding); - - if(status != MJB_STATUS_OK) { - return status; - } - - any_transformation = true; - } - } - if(transformed != NULL) { *transformed = any_transformation; } @@ -153,11 +141,19 @@ static mjb_status mjb_filter_write(mjb_output *output, const void *context) { } MJB_EXPORT mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_filter_flags filters, mjb_encoding output_encoding, mjb_result *result) { - if(result == NULL || (buffer == NULL && byte_length > 0)) { + mjb_malformed_policy malformed_policy, mjb_filter_flags filters, mjb_encoding output_encoding, + mjb_result *result, mjb_diagnostic *diagnostic) { + if(result == NULL || (buffer == NULL && byte_length > 0) || + !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -177,8 +173,8 @@ MJB_EXPORT mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_enc if(filters & MJB_FILTER_NORMALIZE) { mjb_encoding normalize_output_encoding = filters == MJB_FILTER_NORMALIZE ? output_encoding : encoding; - status = mjb_normalize(buffer, byte_length, encoding, MJB_NORMALIZATION_NFC, - normalize_output_encoding, result); + status = mjb_normalize(buffer, byte_length, encoding, malformed_policy, + MJB_NORMALIZATION_NFC, normalize_output_encoding, result, diagnostic); if(status != MJB_STATUS_OK) { return status; @@ -208,7 +204,8 @@ MJB_EXPORT mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_enc mjb_output output; mjb_output_init_dynamic(&output, allocated, byte_length); - mjb_filter_context context = { buffer, byte_length, encoding, filters, output_encoding }; + mjb_filter_context context = { buffer, byte_length, encoding, malformed_policy, filters, + output_encoding, diagnostic }; bool transformed = false; status = mjb_filter_process(&context, &output, &transformed); @@ -247,17 +244,32 @@ MJB_EXPORT mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_enc } MJB_EXPORT mjb_status mjb_filter_into(const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_filter_flags filters, mjb_encoding output_encoding, void *output, size_t *output_size) { + mjb_malformed_policy malformed_policy, mjb_filter_flags filters, mjb_encoding output_encoding, + void *output, size_t *output_size, mjb_diagnostic *diagnostic) { if(output_size == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ARGUMENT; + } + if(buffer == NULL && byte_length > 0) { *output_size = 0; return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -270,17 +282,20 @@ MJB_EXPORT mjb_status mjb_filter_into(const char *buffer, size_t byte_length, mj return mjb_output_copy_into(buffer, byte_length, output, output_size); } + bool normalization_requested = (filters & MJB_FILTER_NORMALIZE) != 0; + if(filters == MJB_FILTER_NORMALIZE) { - return mjb_normalize_into(buffer, byte_length, encoding, MJB_NORMALIZATION_NFC, - output_encoding, output, output_size); + status = mjb_normalize_into(buffer, byte_length, encoding, malformed_policy, + MJB_NORMALIZATION_NFC, output_encoding, output, output_size, diagnostic); + + return status; } mjb_result normalized = { NULL, 0, false }; - bool normalization_requested = (filters & MJB_FILTER_NORMALIZE) != 0; if(normalization_requested) { - status = mjb_normalize(buffer, byte_length, encoding, MJB_NORMALIZATION_NFC, encoding, - &normalized); + status = mjb_normalize(buffer, byte_length, encoding, malformed_policy, + MJB_NORMALIZATION_NFC, encoding, &normalized, diagnostic); if(status != MJB_STATUS_OK) { *output_size = 0; @@ -292,7 +307,8 @@ MJB_EXPORT mjb_status mjb_filter_into(const char *buffer, size_t byte_length, mj byte_length = normalized.output_size; } - mjb_filter_context context = { buffer, byte_length, encoding, filters, output_encoding }; + mjb_filter_context context = { buffer, byte_length, encoding, malformed_policy, filters, + output_encoding, diagnostic }; status = mjb_output_into(output, output_size, mjb_filter_write, &context); if(normalization_requested) { diff --git a/src/idna.c b/src/idna.c index d8c3aba7..c3ece99a 100644 --- a/src/idna.c +++ b/src/idna.c @@ -54,7 +54,7 @@ static mjb_status mjb_idna_decode_utf8(const char *buffer, size_t byte_length, bool in_error = false; mjb_codepoint codepoint = 0; - for(size_t index = 0; index < byte_length;) { + for(size_t index = 0;;) { mjb_decode_result decoded = mjb_next_codepoint(buffer, byte_length, &state, &index, MJB_ENC_UTF_8, &codepoint, &in_error); @@ -104,7 +104,7 @@ static mjb_status mjb_idna_map(const char *buffer, size_t byte_length, mjb_encod bool in_error = false; mjb_codepoint codepoint = 0; - for(size_t index = 0; index < byte_length;) { + for(size_t index = 0;;) { mjb_decode_result decoded = mjb_next_codepoint(buffer, byte_length, &state, &index, encoding, &codepoint, &in_error); @@ -408,8 +408,8 @@ static bool mjb_idna_valid_contextj(const mjb_idna_codepoints *label) { static mjb_status mjb_idna_label_is_nfc(const char *label, size_t byte_length, bool *is_nfc) { mjb_result normalized; - mjb_status status = mjb_normalize(label, byte_length, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, &normalized); + mjb_status status = mjb_normalize(label, byte_length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &normalized, NULL); if(status != MJB_STATUS_OK) { return status; @@ -632,7 +632,7 @@ static mjb_status mjb_idna_finish_output(mjb_output *output, mjb_encoding output } mjb_status status = mjb_convert_encoding(output->buffer, output->size, MJB_ENC_UTF_8, - output_encoding, result); + MJB_MALFORMED_STOP, output_encoding, result, NULL); mjb_free(output->buffer); output->buffer = NULL; @@ -660,6 +660,12 @@ static mjb_status mjb_idna_process(const char *buffer, size_t byte_length, mjb_e return status; } + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); + + if(status != MJB_STATUS_OK) { + return status; + } + if(encoding == MJB_ENC_ASCII) { for(size_t i = 0; i < byte_length; ++i) { if(((uint8_t)buffer[i] & 0x80) != 0) { @@ -668,12 +674,6 @@ static mjb_status mjb_idna_process(const char *buffer, size_t byte_length, mjb_e } } - status = mjb_validate_code_unit_sequence(buffer, byte_length, encoding); - - if(status != MJB_STATUS_OK) { - return status; - } - mjb_output mapped; status = mjb_idna_map(buffer, byte_length, encoding, &mapped); @@ -682,8 +682,8 @@ static mjb_status mjb_idna_process(const char *buffer, size_t byte_length, mjb_e } mjb_result normalized; - status = mjb_normalize(mapped.buffer, mapped.size, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, &normalized); + status = mjb_normalize(mapped.buffer, mapped.size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &normalized, NULL); if(status != MJB_STATUS_OK) { mjb_free(mapped.buffer); diff --git a/src/locales.c b/src/locales.c index 516867e0..6aa9e0e4 100644 --- a/src/locales.c +++ b/src/locales.c @@ -503,7 +503,8 @@ MJB_EXPORT mjb_status mjb_locale_parse(const char *id, size_t size, mjb_encoding } else if(encoding == MJB_ENC_UTF_8 && mjb_is_ascii(id, size)) { // Already suitable for the byte-oriented locale parser. } else { - status = mjb_convert_encoding(id, size, encoding, MJB_ENC_ASCII, &converted); + status = mjb_convert_encoding(id, size, encoding, MJB_MALFORMED_STOP, MJB_ENC_ASCII, + &converted, NULL); if(status == MJB_STATUS_OK) { ascii_id = converted.output; diff --git a/src/mojibake-internal.h b/src/mojibake-internal.h index 2a9962d9..390d11fa 100644 --- a/src/mojibake-internal.h +++ b/src/mojibake-internal.h @@ -88,6 +88,10 @@ MJB_LOCAL void *mjb_realloc(void *ptr, size_t new_size); MJB_LOCAL void mjb_free(void *ptr); +MJB_LOCAL mjb_status mjb_normalization_quick_check_internal(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_normalization form, mjb_quick_check_result *quick_check, + bool validate_full_input); + char *mjb_string_output(char *ret, char *input, size_t input_size, size_t *output_index, size_t *output_size); diff --git a/src/mojibake.c b/src/mojibake.c index 00c0fe55..bc831c64 100644 --- a/src/mojibake.c +++ b/src/mojibake.c @@ -113,7 +113,9 @@ MJB_EXPORT MJB_CONST const char *mjb_status_message(mjb_status status) { // MJB_STATUS_NOT_FOUND, "No Unicode data was found for the requested value", // MJB_STATUS_FEATURE_NOT_ENABLED - "The requested feature was disabled when the library was built" + "The requested feature was disabled when the library was built", + // MJB_STATUS_END_OF_INPUT + "The end of the input was reached" }; if((unsigned int)status < (sizeof(messages) / sizeof(messages[0]))) { diff --git a/src/mojibake.h b/src/mojibake.h index f0b53683..c38f3b78 100644 --- a/src/mojibake.h +++ b/src/mojibake.h @@ -311,9 +311,40 @@ typedef enum mjb_status { MJB_STATUS_OUTPUT_TOO_SMALL, MJB_STATUS_CALLBACK_STOPPED, MJB_STATUS_NOT_FOUND, - MJB_STATUS_FEATURE_NOT_ENABLED + MJB_STATUS_FEATURE_NOT_ENABLED, + MJB_STATUS_END_OF_INPUT } mjb_status; +// Policy used by decoding operations when an ill-formed code-unit sequence is encountered. +typedef enum mjb_malformed_policy { + MJB_MALFORMED_STOP, // Report MJB_STATUS_MALFORMED_INPUT + MJB_MALFORMED_REPLACE, // Emit one U+FFFD per maximal ill-formed subsequence + MJB_MALFORMED_SKIP // Discard maximal ill-formed subsequences +} mjb_malformed_policy; + +// Precise reason that a code-unit sequence is not well-formed. +typedef enum mjb_text_error { + MJB_TEXT_ERROR_NONE = 0, + MJB_TEXT_ERROR_NON_ASCII, + MJB_TEXT_ERROR_INVALID_LEADING_BYTE, + MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + MJB_TEXT_ERROR_MISSING_CONTINUATION, + MJB_TEXT_ERROR_OVERLONG_SEQUENCE, + MJB_TEXT_ERROR_SURROGATE, + MJB_TEXT_ERROR_OUT_OF_RANGE, + MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, + MJB_TEXT_ERROR_TRUNCATED_CODE_UNIT, + MJB_TEXT_ERROR_UNPAIRED_SURROGATE +} mjb_text_error; + +// Location and extent of the first malformed subsequence encountered by an operation. +typedef struct mjb_diagnostic { + mjb_text_error error; + size_t byte_offset; + size_t byte_length; + size_t code_unit_offset; +} mjb_diagnostic; + typedef struct mjb_locale_id { char language[9]; char extlang[12]; @@ -468,6 +499,7 @@ typedef struct mjb_next_state { mjb_codepoint previous_codepoint; mjb_codepoint current_codepoint; bool in_error; + bool had_error; unsigned short ri_count; bool ext_pict_seen; bool zwj_seen; @@ -506,6 +538,7 @@ typedef struct mjb_next_word_state { mjb_codepoint current_codepoint; mjb_wbp prev_prev_wbp; bool in_error; + bool had_error; unsigned short ri_count; bool wb4_merged; bool zwj_pending; @@ -521,6 +554,7 @@ typedef struct mjb_next_sentence_state { mjb_codepoint previous_codepoint; mjb_codepoint current_codepoint; bool in_error; + bool had_error; bool sb5_merged; bool in_sat; bool sat_has_sp; @@ -591,22 +625,22 @@ typedef enum mjb_script_set_kind { MJB_EXPORT MJB_NODISCARD mjb_status mjb_codepoint_info(mjb_codepoint codepoint, mjb_character *character); // Normalize a string to NFC/NFKC/NFD/NFKD form. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_normalization form, mjb_encoding output_encoding, mjb_result *result); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_normalization form, mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic); // Normalize a string into a caller-provided buffer. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_normalization form, mjb_encoding output_encoding, void *output, size_t *output_size); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_normalization form, mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic); // Filter a string with the selected mjb_filter_flags. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_filter_flags filters, mjb_encoding output_encoding, mjb_result *result); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_filter(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_filter_flags filters, mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic); // Filter a string into a caller-provided buffer. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_filter_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_filter_flags filters, mjb_encoding output_encoding, void *output, size_t *output_size); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_filter_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_filter_flags filters, mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic); // Apply the Unicode NFKC_Casefold transform to a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_nfkc_casefold(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_encoding output_encoding, mjb_result *result); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_nfkc_casefold(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic); // Apply the Unicode NFKC_Casefold transform into a caller-provided buffer. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_nfkc_casefold_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_encoding output_encoding, void *output, size_t *output_size); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_nfkc_casefold_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic); // Convert a domain name to its UTS #46 nontransitional ASCII form. MJB_EXPORT MJB_NODISCARD mjb_status mjb_idna_to_ascii(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_encoding output_encoding, mjb_idna_info *info, mjb_result *result); @@ -635,11 +669,20 @@ MJB_EXPORT MJB_PURE bool mjb_is_utf8(const char *buffer, size_t byte_length); // Return true if the string is encoded in UTF-16BE or UTF-16LE. MJB_EXPORT MJB_PURE bool mjb_is_utf16(const char *buffer, size_t byte_length); +// Validate a complete Unicode code-unit sequence. +MJB_EXPORT MJB_NODISCARD mjb_status mjb_string_validate(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_diagnostic *diagnostic); + +// Decode the next codepoint from a string. +MJB_EXPORT MJB_NODISCARD mjb_status mjb_decode_next(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *offset, mjb_codepoint *codepoint, mjb_diagnostic *diagnostic); + +// Decode the previous codepoint from a string. +MJB_EXPORT MJB_NODISCARD mjb_status mjb_decode_previous(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *offset, mjb_codepoint *codepoint, mjb_diagnostic *diagnostic); + // Count the codepoints in a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_codepoint_count(const char *buffer, size_t byte_length, mjb_encoding encoding, size_t *count); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_codepoint_count(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic); // Run a callback for each codepoint of a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_for_each_codepoint(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_for_each_codepoint_fn callback); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_for_each_codepoint(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_for_each_codepoint_fn callback, mjb_diagnostic *diagnostic); // Return the value of a binary Unicode property. MJB_EXPORT MJB_NODISCARD mjb_status mjb_codepoint_property_binary(mjb_codepoint codepoint, mjb_property property, bool *value); @@ -663,10 +706,10 @@ MJB_EXPORT MJB_NODISCARD mjb_status mjb_codepoint_script_extensions(mjb_codepoin MJB_EXPORT unsigned int mjb_codepoint_encode(mjb_codepoint codepoint, char *buffer, size_t byte_length, mjb_encoding encoding); // Convert from one encoding to another. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_convert_encoding(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_encoding output_encoding, mjb_result *result); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_convert_encoding(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic); // Convert from one encoding to another into a caller-provided buffer. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_convert_encoding_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_encoding output_encoding, void *output, size_t *output_size); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_convert_encoding_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic); // Compare two strings using a Unicode caseless matching relation. MJB_EXPORT MJB_NODISCARD mjb_status mjb_caseless_match(const char *s1, size_t s1_byte_length, mjb_encoding s1_encoding, const char *s2, size_t s2_byte_length, mjb_encoding s2_encoding, mjb_caseless_mode mode, bool *matches); @@ -675,16 +718,16 @@ MJB_EXPORT MJB_NODISCARD mjb_status mjb_caseless_match(const char *s1, size_t s1 MJB_EXPORT MJB_NODISCARD mjb_status mjb_collation_compare(const char *s1, size_t s1_byte_length, mjb_encoding s1_encoding, const char *s2, size_t s2_byte_length, mjb_encoding s2_encoding, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, int *order); // Generate a UCA sort key for a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_collation_key(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, mjb_result *result); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_collation_key(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, mjb_result *result, mjb_diagnostic *diagnostic); // Generate a binary collation key into a caller-provided buffer. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_collation_key_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, void *output, size_t *output_size); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_collation_key_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_collation_variable_weighting variable_weighting, mjb_collation_strength strength, void *output, size_t *output_size, mjb_diagnostic *diagnostic); // Change string case. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_map_case(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_map_case_type type, mjb_encoding output_encoding, mjb_result *result); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_map_case(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_map_case_type type, mjb_encoding output_encoding, mjb_result *result, mjb_diagnostic *diagnostic); // Change string case into a caller-provided buffer. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_map_case_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_map_case_type type, mjb_encoding output_encoding, void *output, size_t *output_size); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_map_case_into(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_map_case_type type, mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic); // Return true if the codepoint is valid. MJB_EXPORT MJB_CONST bool mjb_codepoint_is_valid(mjb_codepoint codepoint); @@ -732,7 +775,7 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le MJB_EXPORT mjb_break_type mjb_next_sentence_break(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_next_sentence_state *state); // Count the sentence segments in a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_sentence_count(const char *buffer, size_t byte_length, mjb_encoding encoding, size_t *count); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_sentence_count(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic); // Grapheme cluster breaking. MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_next_state *state); @@ -741,7 +784,7 @@ MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byt MJB_EXPORT size_t mjb_truncate_grapheme(const char *buffer, size_t byte_length, mjb_encoding encoding, size_t max_graphemes); // Count the extended grapheme clusters in a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_grapheme_count(const char *buffer, size_t byte_length, mjb_encoding encoding, size_t *count); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_grapheme_count(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic); // Return the number of bytes whose grapheme clusters fit within max_columns terminal cells. MJB_EXPORT size_t mjb_truncate_grapheme_width(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_terminal_width_profile profile, size_t max_columns); @@ -750,7 +793,7 @@ MJB_EXPORT size_t mjb_truncate_grapheme_width(const char *buffer, size_t byte_le MJB_EXPORT size_t mjb_truncate_word(const char *buffer, size_t byte_length, mjb_encoding encoding, size_t max_segments); // Count the word-like segments in a string. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_word_count(const char *buffer, size_t byte_length, mjb_encoding encoding, size_t *count); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_word_count(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, mjb_diagnostic *diagnostic); // Return the number of bytes whose word-break segments fit within max_columns terminal cells. MJB_EXPORT size_t mjb_truncate_word_width(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_terminal_width_profile profile, size_t max_columns); @@ -855,7 +898,7 @@ MJB_EXPORT size_t mjb_hangul_syllable_composition(mjb_buffer_character *characte MJB_EXPORT MJB_NODISCARD mjb_status mjb_codepoint_east_asian_width(mjb_codepoint codepoint, mjb_east_asian_width *width); // Return the estimated terminal-cell width of printable, single-line text. -MJB_EXPORT MJB_NODISCARD mjb_status mjb_terminal_width(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_terminal_width_profile profile, size_t *width); +MJB_EXPORT MJB_NODISCARD mjb_status mjb_terminal_width(const char *buffer, size_t byte_length, mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_terminal_width_profile profile, size_t *width, mjb_diagnostic *diagnostic); // Parse a BCP 47 language tag. MJB_EXPORT MJB_NODISCARD mjb_status mjb_locale_parse(const char *id, size_t byte_length, mjb_encoding encoding, mjb_locale_id *locale); diff --git a/src/next.c b/src/next.c index 4e5cf32e..86feca7c 100644 --- a/src/next.c +++ b/src/next.c @@ -11,21 +11,24 @@ * Run a callback for each codepoint in the string. */ MJB_EXPORT mjb_status mjb_for_each_codepoint(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_for_each_codepoint_fn callback) { - if(buffer == NULL || byte_length == 0) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_for_each_codepoint_fn callback, mjb_diagnostic *diagnostic) { + if((buffer == NULL && byte_length > 0) || !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { return status; } - if(byte_length == 0) { - return MJB_STATUS_INVALID_ARGUMENT; - } - #ifndef __EMSCRIPTEN__ // Emscripten uses _mjbForEachCodepointCallback. if(callback == NULL) { @@ -33,29 +36,27 @@ MJB_EXPORT mjb_status mjb_for_each_codepoint(const char *buffer, size_t byte_len } #endif - uint8_t state = MJB_UTF_ACCEPT; - bool in_error = false; mjb_codepoint codepoint = 0; mjb_character character; bool has_previous_character = false; bool first_character = true; - // Loop through the string. - for(size_t i = 0; i < byte_length;) { - // Find next codepoint. - mjb_decode_result result = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, - &codepoint, &in_error); + size_t offset = 0; + + for(;;) { + mjb_diagnostic current; + mjb_status result = mjb_decode_next(buffer, byte_length, encoding, malformed_policy, + &offset, &codepoint, ¤t); + mjb_diagnostic_record(diagnostic, ¤t); - if(result == MJB_DECODE_END) { + if(result == MJB_STATUS_END_OF_INPUT) { break; } - if(result == MJB_DECODE_INCOMPLETE) { - continue; + if(result != MJB_STATUS_OK) { + return result; } - // result is MJB_DECODE_OK or MJB_DECODE_ERROR (both have valid codepoint) - if(has_previous_character) { #ifdef __EMSCRIPTEN__ bool asm_result = EM_ASM_INT( diff --git a/src/normalization.c b/src/normalization.c index 65d90d27..a5f890c1 100644 --- a/src/normalization.c +++ b/src/normalization.c @@ -246,7 +246,7 @@ static mjb_status mjb_normalization_estimate(const char *buffer, size_t byte_len *potential_output_size = byte_length; } else { mjb_status count_status = mjb_codepoint_count(buffer, byte_length, encoding, - potential_output_size); + MJB_MALFORMED_STOP, potential_output_size, NULL); if(count_status != MJB_STATUS_OK) { return count_status; @@ -354,7 +354,7 @@ static mjb_status mjb_normalize_write(mjb_output *output, const void *context_po // Loop through the string. bool in_error = false; - for(size_t i = 0; i < byte_length;) { + for(size_t i = 0;;) { // Find next codepoint. mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, &codepoint, &in_error); @@ -367,6 +367,12 @@ static mjb_status mjb_normalize_write(mjb_output *output, const void *context_po continue; } + if(decode_status == MJB_DECODE_ERROR) { + status = MJB_STATUS_MALFORMED_INPUT; + + goto fail; + } + // Get current character. if(!mjb_n_codepoint_character(codepoint, ¤t_character)) { continue; @@ -531,15 +537,23 @@ static mjb_status mjb_normalize_write(mjb_output *output, const void *context_po * Normalize a string */ MJB_EXPORT mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_encoding encoding, - mjb_normalization form, mjb_encoding output_encoding, mjb_result *result) { - if(result == NULL || (buffer == NULL && byte_length > 0)) { + mjb_malformed_policy malformed_policy, mjb_normalization form, mjb_encoding output_encoding, + mjb_result *result, mjb_diagnostic *diagnostic) { + if(result == NULL || (buffer == NULL && byte_length > 0) || + !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + if(!mjb_normalization_form_is_valid(form)) { return MJB_STATUS_INVALID_FORM; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -558,20 +572,44 @@ MJB_EXPORT mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_ result->output_size = 0; result->transformed = false; + mjb_result sanitized = { NULL, 0, false }; mjb_quick_check_result is_normalized; - status = mjb_normalization_quick_check(buffer, byte_length, encoding, form, &is_normalized); + status = mjb_normalization_quick_check_internal(buffer, byte_length, encoding, form, + &is_normalized, false); + + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); + + if(status == MJB_STATUS_OK && byte_length > 0) { + status = mjb_normalization_quick_check_internal(buffer, byte_length, encoding, form, + &is_normalized, false); + } + } if(status != MJB_STATUS_OK) { return status; } + if(byte_length == 0) { + *result = sanitized; + + return MJB_STATUS_OK; + } + if(is_normalized == MJB_QC_YES) { if(encoding != output_encoding) { - return mjb_convert_encoding(buffer, byte_length, encoding, output_encoding, result); + status = mjb_convert_encoding(buffer, byte_length, encoding, MJB_MALFORMED_STOP, + output_encoding, result, NULL); + mjb_result_free(&sanitized); + + return status; } result->output = (char *)buffer; result->output_size = byte_length; + result->transformed = sanitized.transformed; + sanitized.transformed = false; return MJB_STATUS_OK; } @@ -581,12 +619,16 @@ MJB_EXPORT mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_ &potential_output_size); if(status != MJB_STATUS_OK) { + mjb_result_free(&sanitized); + return status; } char *allocated = (char *)mjb_alloc(potential_output_size); if(allocated == NULL) { + mjb_result_free(&sanitized); + return MJB_STATUS_NO_MEMORY; } @@ -597,11 +639,14 @@ MJB_EXPORT mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_ status = mjb_normalize_write(&output, &context); if(status != MJB_STATUS_OK) { + mjb_diagnose_malformed_input(buffer, byte_length, encoding, status, diagnostic); mjb_free(output.buffer); + mjb_result_free(&sanitized); return status == MJB_STATUS_UNSUPPORTED ? MJB_STATUS_NO_MEMORY : status; } + mjb_result_free(&sanitized); output.buffer[output.size] = '\0'; result->output = output.buffer; result->output_size = output.size; @@ -611,13 +656,15 @@ MJB_EXPORT mjb_status mjb_normalize(const char *buffer, size_t byte_length, mjb_ } MJB_EXPORT mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_normalization form, mjb_encoding output_encoding, void *output, - size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_normalization form, + mjb_encoding output_encoding, void *output, size_t *output_size, mjb_diagnostic *diagnostic) { if(output_size == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } - if(buffer == NULL && byte_length > 0) { + mjb_diagnostic_reset(diagnostic); + + if((buffer == NULL && byte_length > 0) || !mjb_malformed_policy_is_valid(malformed_policy)) { *output_size = 0; return MJB_STATUS_INVALID_ARGUMENT; @@ -629,6 +676,12 @@ MJB_EXPORT mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, return MJB_STATUS_INVALID_FORM; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -641,8 +694,20 @@ MJB_EXPORT mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, return mjb_output_copy_into(buffer, byte_length, output, output_size); } + mjb_result sanitized = { NULL, 0, false }; mjb_quick_check_result is_normalized; - status = mjb_normalization_quick_check(buffer, byte_length, encoding, form, &is_normalized); + status = mjb_normalization_quick_check_internal(buffer, byte_length, encoding, form, + &is_normalized, false); + + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); + + if(status == MJB_STATUS_OK && byte_length > 0) { + status = mjb_normalization_quick_check_internal(buffer, byte_length, encoding, form, + &is_normalized, false); + } + } if(status != MJB_STATUS_OK) { *output_size = 0; @@ -650,13 +715,24 @@ MJB_EXPORT mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, return status; } + if(byte_length == 0) { + status = mjb_output_copy_into(buffer, byte_length, output, output_size); + mjb_result_free(&sanitized); + + return status; + } + if(is_normalized == MJB_QC_YES) { if(encoding != output_encoding) { - return mjb_convert_encoding_into(buffer, byte_length, encoding, output_encoding, output, - output_size); + status = mjb_convert_encoding_into(buffer, byte_length, encoding, MJB_MALFORMED_STOP, + output_encoding, output, output_size, NULL); + } else { + status = mjb_output_copy_into(buffer, byte_length, output, output_size); } - return mjb_output_copy_into(buffer, byte_length, output, output_size); + mjb_result_free(&sanitized); + + return status; } size_t potential_output_size = 0; @@ -665,6 +741,7 @@ MJB_EXPORT mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, if(status != MJB_STATUS_OK) { *output_size = 0; + mjb_result_free(&sanitized); return status; } @@ -672,7 +749,10 @@ MJB_EXPORT mjb_status mjb_normalize_into(const char *buffer, size_t byte_length, mjb_normalize_context context = { buffer, byte_length, potential_output_size, encoding, form, output_encoding }; - return mjb_output_into(output, output_size, mjb_normalize_write, &context); + status = mjb_output_into(output, output_size, mjb_normalize_write, &context); + mjb_result_free(&sanitized); + + return status; } // Apply full default case folding and remove Default_Ignorable_Code_Point characters. @@ -756,8 +836,8 @@ static mjb_status mjb_nfkc_casefold_pass(const char *buffer, size_t byte_length, } static mjb_status mjb_nfkc_casefold_transform(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_encoding output_encoding, mjb_result *result, void *output, - size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, + mjb_result *result, void *output, size_t *output_size, mjb_diagnostic *diagnostic) { const char *current = buffer; size_t current_size = byte_length; mjb_encoding current_encoding = encoding; @@ -769,7 +849,8 @@ static mjb_status mjb_nfkc_casefold_transform(const char *buffer, size_t byte_le mjb_result normalized; // We use UTF-8 as the intermediate encoding for the NFKC_Casefold transform. mjb_status status = mjb_normalize(current, current_size, current_encoding, - MJB_NORMALIZATION_NFKC, MJB_ENC_UTF_8, &normalized); + pass == 0 ? malformed_policy : MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFKC, + MJB_ENC_UTF_8, &normalized, pass == 0 ? diagnostic : NULL); if(status != MJB_STATUS_OK) { if(current_owned) { @@ -811,8 +892,8 @@ static mjb_status mjb_nfkc_casefold_transform(const char *buffer, size_t byte_le if(stable) { if(result != NULL) { mjb_result normalized_result; - status = mjb_normalize(current, current_size, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - output_encoding, &normalized_result); + status = mjb_normalize(current, current_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, output_encoding, &normalized_result, NULL); if(status != MJB_STATUS_OK) { mjb_free((void *)current); @@ -830,8 +911,8 @@ static mjb_status mjb_nfkc_casefold_transform(const char *buffer, size_t byte_le return MJB_STATUS_OK; } - status = mjb_normalize_into(current, current_size, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - output_encoding, output, output_size); + status = mjb_normalize_into(current, current_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, output_encoding, output, output_size, NULL); mjb_free((void *)current); return status; @@ -847,11 +928,19 @@ static mjb_status mjb_nfkc_casefold_transform(const char *buffer, size_t byte_le * Apply the Unicode NFKC_Casefold transform without duplicating its derived mapping table. */ MJB_EXPORT mjb_status mjb_nfkc_casefold(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_encoding output_encoding, mjb_result *result) { - if(result == NULL || (buffer == NULL && byte_length > 0)) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, + mjb_result *result, mjb_diagnostic *diagnostic) { + if(result == NULL || (buffer == NULL && byte_length > 0) || + !mjb_malformed_policy_is_valid(malformed_policy)) { return MJB_STATUS_INVALID_ARGUMENT; } + mjb_diagnostic_reset(diagnostic); + + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -866,22 +955,33 @@ MJB_EXPORT mjb_status mjb_nfkc_casefold(const char *buffer, size_t byte_length, return MJB_STATUS_OK; } - return mjb_nfkc_casefold_transform(buffer, byte_length, encoding, output_encoding, result, NULL, - NULL); + status = mjb_nfkc_casefold_transform(buffer, byte_length, encoding, malformed_policy, + output_encoding, result, NULL, NULL, diagnostic); + + return status; } MJB_EXPORT mjb_status mjb_nfkc_casefold_into(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_encoding output_encoding, void *output, size_t *output_size) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, mjb_encoding output_encoding, + void *output, size_t *output_size, mjb_diagnostic *diagnostic) { if(output_size == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } - if(buffer == NULL && byte_length > 0) { + mjb_diagnostic_reset(diagnostic); + + if((buffer == NULL && byte_length > 0) || !mjb_malformed_policy_is_valid(malformed_policy)) { *output_size = 0; return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding) || !mjb_encoding_is_valid_output(output_encoding)) { + *output_size = 0; + + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { @@ -894,8 +994,8 @@ MJB_EXPORT mjb_status mjb_nfkc_casefold_into(const char *buffer, size_t byte_len return mjb_output_copy_into(buffer, byte_length, output, output_size); } - status = mjb_nfkc_casefold_transform(buffer, byte_length, encoding, output_encoding, NULL, - output, output_size); + status = mjb_nfkc_casefold_transform(buffer, byte_length, encoding, malformed_policy, + output_encoding, NULL, output, output_size, diagnostic); if(status != MJB_STATUS_OK && status != MJB_STATUS_OUTPUT_TOO_SMALL) { *output_size = 0; diff --git a/src/quick-check.c b/src/quick-check.c index ee1d05a6..f68f8aae 100644 --- a/src/quick-check.c +++ b/src/quick-check.c @@ -13,8 +13,9 @@ extern mojibake mjb_global; * Check if a string is normalized to NFC/NFKC/NFD/NFKD form. * See: https://unicode.org/reports/tr15/#Detecting_Normalization_Forms */ -MJB_EXPORT mjb_status mjb_normalization_quick_check(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_normalization form, mjb_quick_check_result *quick_check) { +MJB_LOCAL mjb_status mjb_normalization_quick_check_internal(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_normalization form, mjb_quick_check_result *quick_check, + bool validate_full_input) { if(quick_check == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } @@ -62,7 +63,16 @@ MJB_EXPORT mjb_status mjb_normalization_quick_check(const char *buffer, size_t b mjb_n_character current_character; bool in_error = false; - for(size_t i = 0; i < byte_length;) { +#define MJB_QUICK_CHECK_NO() \ + do { \ + result = MJB_QC_NO; \ + if(!validate_full_input) { \ + *quick_check = result; \ + return MJB_STATUS_OK; \ + } \ + } while(0) + + for(size_t i = 0;;) { // Find next codepoint. mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, &codepoint, &in_error); @@ -97,15 +107,11 @@ MJB_EXPORT mjb_status mjb_normalization_quick_check(const char *buffer, size_t b if(last_canonical_class > current_character.combining && current_character.combining != MJB_CCC_NOT_REORDERED) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } if(current_character.quick_check == MJB_QC_NO) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } bool is_hangul_syllable = mjb_codepoint_is_hangul_syllable(codepoint); @@ -113,51 +119,43 @@ MJB_EXPORT mjb_status mjb_normalization_quick_check(const char *buffer, size_t b switch(form) { case MJB_NORMALIZATION_NFC: if(current_character.quick_check & MJB_QC_NFC_MAYBE) { - result = MJB_QC_MAYBE; + if(result == MJB_QC_YES) { + result = MJB_QC_MAYBE; + } } else if(current_character.quick_check & MJB_QC_NFC_NO) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } break; case MJB_NORMALIZATION_NFKC: if(current_character.quick_check & MJB_QC_NFKC_MAYBE) { - result = MJB_QC_MAYBE; + if(result == MJB_QC_YES) { + result = MJB_QC_MAYBE; + } } else if(current_character.quick_check & MJB_QC_NFKC_NO) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } break; case MJB_NORMALIZATION_NFD: if(is_hangul_syllable) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } // There are no MAYBE values for NFD. if(current_character.quick_check & MJB_QC_NFD_NO) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } break; case MJB_NORMALIZATION_NFKD: if(is_hangul_syllable) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } // There are no MAYBE values for NFKD. if(current_character.quick_check & MJB_QC_NFKD_NO) { - *quick_check = MJB_QC_NO; - - return MJB_STATUS_OK; + MJB_QUICK_CHECK_NO(); } break; @@ -172,5 +170,13 @@ MJB_EXPORT mjb_status mjb_normalization_quick_check(const char *buffer, size_t b *quick_check = result; +#undef MJB_QUICK_CHECK_NO + return MJB_STATUS_OK; } + +MJB_EXPORT mjb_status mjb_normalization_quick_check(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_normalization form, mjb_quick_check_result *quick_check) { + return mjb_normalization_quick_check_internal(buffer, byte_length, encoding, form, quick_check, + true); +} diff --git a/src/security.c b/src/security.c index a54d44a1..25ce9515 100644 --- a/src/security.c +++ b/src/security.c @@ -104,13 +104,17 @@ MJB_EXPORT mjb_status mjb_resolved_script_set(const char *buffer, size_t byte_le return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK) { return status; } - status = mjb_validate_code_unit_sequence(buffer, byte_length, encoding); + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); if(status != MJB_STATUS_OK) { return status; @@ -123,7 +127,7 @@ MJB_EXPORT mjb_status mjb_resolved_script_set(const char *buffer, size_t byte_le bool in_error = false; mjb_codepoint codepoint = 0; - for(size_t i = 0; i < byte_length;) { + for(size_t i = 0;;) { mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, &codepoint, &in_error); @@ -227,8 +231,8 @@ static mjb_status mjb_confusable_skeleton_finish(const char *buffer, size_t byte // Step 1: NFD the input. mjb_result nfd; - mjb_status status = mjb_normalize(buffer, byte_length, encoding, MJB_NORMALIZATION_NFD, - MJB_ENC_UTF_8, &nfd); + mjb_status status = mjb_normalize(buffer, byte_length, encoding, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, &nfd, NULL); if(status != MJB_STATUS_OK) { return status; @@ -309,8 +313,8 @@ static mjb_status mjb_confusable_skeleton_finish(const char *buffer, size_t byte // Step 3: NFD the intermediate string into the selected final output. if(result != NULL) { - status = mjb_normalize(mid, mid_index, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFD, - output_encoding, result); + status = mjb_normalize(mid, mid_index, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFD, output_encoding, result, NULL); // mjb_normalize may return result->output == mid when the string is already NFD. // In that case transfer ownership so the caller can free it via mjb_result_free. @@ -323,8 +327,8 @@ static mjb_status mjb_confusable_skeleton_finish(const char *buffer, size_t byte return status; } - status = mjb_normalize_into(mid, mid_index, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFD, - output_encoding, output, output_size); + status = mjb_normalize_into(mid, mid_index, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFD, output_encoding, output, output_size, NULL); mjb_free(mid); return status; @@ -343,12 +347,6 @@ static mjb_status mjb_confusable_skeleton_process(const char *buffer, size_t byt return status; } - status = mjb_validate_code_unit_sequence(buffer, byte_length, encoding); - - if(status != MJB_STATUS_OK) { - return status; - } - mjb_bidi_paragraph paragraph; status = mjb_bidi_resolve(buffer, byte_length, encoding, MJB_DIRECTION_LTR, ¶graph); diff --git a/src/segmentation.c b/src/segmentation.c index 21b3dd65..31aa3731 100644 --- a/src/segmentation.c +++ b/src/segmentation.c @@ -53,6 +53,7 @@ MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byt state->previous_codepoint = MJB_CODEPOINT_NOT_VALID; state->current_codepoint = MJB_CODEPOINT_NOT_VALID; state->in_error = false; + state->had_error = false; state->ri_count = 0; state->ext_pict_seen = false; state->zwj_seen = false; @@ -66,6 +67,10 @@ MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byt if(state->index == byte_length) { // Reached end of string. + if(mjb_utf_state_is_incomplete(state->state)) { + state->had_error = true; + } + ++state->index; // GB2 Any ÷ eot @@ -83,6 +88,10 @@ MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byt mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state->state, &state->index, encoding, &codepoint, &state->in_error); + if(decode_status == MJB_DECODE_ERROR) { + state->had_error = true; + } + if(decode_status == MJB_DECODE_END) { mjb_mark_decode_terminated(&state->state, &state->index, &state->current_codepoint, encoding); @@ -243,6 +252,10 @@ MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byt return MJB_BT_ALLOWED; } + if(mjb_utf_state_is_incomplete(state->state)) { + state->had_error = true; + } + ++state->index; return MJB_BT_ALLOWED; @@ -320,14 +333,53 @@ MJB_EXPORT size_t mjb_truncate_grapheme(const char *buffer, size_t byte_length, return state.state == MJB_UTF_TERMINATED ? last_break : byte_length; } +static mjb_status mjb_grapheme_count_process(const char *buffer, size_t byte_length, + mjb_encoding encoding, size_t *count) { + if(byte_length == 0) { + *count = 0; + + return MJB_STATUS_OK; + } + + mjb_next_state state; + state.index = 0; + + mjb_break_type bt; + size_t cluster_count = 0; + + while((bt = mjb_next_grapheme_break(buffer, byte_length, encoding, &state)) != MJB_BT_NOT_SET) { + if(bt != MJB_BT_NO_BREAK) { + ++cluster_count; + } + } + + if(state.had_error) { + return MJB_STATUS_MALFORMED_INPUT; + } + + *count = cluster_count; + + return MJB_STATUS_OK; +} + // Count the extended grapheme clusters in a string. MJB_EXPORT mjb_status mjb_grapheme_count(const char *buffer, size_t byte_length, - mjb_encoding encoding, size_t *count) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, + mjb_diagnostic *diagnostic) { if(count == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } *count = 0; + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } if(byte_length == 0) { return MJB_STATUS_OK; @@ -337,33 +389,33 @@ MJB_EXPORT mjb_status mjb_grapheme_count(const char *buffer, size_t byte_length, return MJB_STATUS_INVALID_ARGUMENT; } - if(!mjb_encoding_is_valid_input(encoding)) { - return MJB_STATUS_INVALID_ENCODING; - } - mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK || byte_length == 0) { return status; } - mjb_next_state state; - state.index = 0; + status = mjb_check_input_encoding_byte_order(buffer, byte_length, encoding); - mjb_break_type bt; - size_t cluster_count = 0; + if(status != MJB_STATUS_OK) { + return status; + } - while((bt = mjb_next_grapheme_break(buffer, byte_length, encoding, &state)) != MJB_BT_NOT_SET) { - if(bt == MJB_BT_NO_BREAK) { - continue; - } + status = mjb_grapheme_count_process(buffer, byte_length, encoding, count); + mjb_result sanitized = { NULL, 0, false }; + + if(status == MJB_STATUS_MALFORMED_INPUT) { + status = mjb_repair_text_input(&buffer, &byte_length, &encoding, malformed_policy, + diagnostic, &sanitized); - ++cluster_count; + if(status == MJB_STATUS_OK) { + status = mjb_grapheme_count_process(buffer, byte_length, encoding, count); + } } - *count = cluster_count; + mjb_result_free(&sanitized); - return MJB_STATUS_OK; + return status; } // Return the number of bytes whose grapheme clusters fit within max_columns terminal cells. @@ -394,8 +446,8 @@ MJB_EXPORT size_t mjb_truncate_grapheme_width(const char *buffer, size_t byte_le state.current_codepoint, encoding, state.state == MJB_UTF_TERMINATED, prev_break); size_t cluster_width = 0; - if(mjb_terminal_width(buffer + prev_break, break_pos - prev_break, encoding, profile, - &cluster_width) != MJB_STATUS_OK) { + if(mjb_terminal_width(buffer + prev_break, break_pos - prev_break, encoding, + MJB_MALFORMED_STOP, profile, &cluster_width, NULL) != MJB_STATUS_OK) { return prev_break; } diff --git a/src/shell/commands/break.c b/src/shell/commands/break.c index 838a89e4..0775ab91 100644 --- a/src/shell/commands/break.c +++ b/src/shell/commands/break.c @@ -298,12 +298,13 @@ static void mjbsh_print_break_analysis(const char *input, mjbsh_break_mode mode) size_t input_size = strlen(input); size_t input_real_size = 0; - if(mjb_codepoint_count(input, input_size, MJB_ENC_UTF_8, &input_real_size) != MJB_STATUS_OK) { + if(mjb_codepoint_count(input, input_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &input_real_size, + NULL) != MJB_STATUS_OK) { input_real_size = 0; } size_t terminal_width = 0; mjb_status terminal_width_status = mjb_terminal_width(input, input_size, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &terminal_width); + MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &terminal_width, NULL); if(cmd_output_mode == OUTPUT_MODE_JSON) { mjbsh_print_break_json(input, input_size, input_real_size, terminal_width, diff --git a/src/shell/commands/character.c b/src/shell/commands/character.c index 37cb8c5a..6268f19c 100644 --- a/src/shell/commands/character.c +++ b/src/shell/commands/character.c @@ -277,7 +277,7 @@ static bool mjbsh_output_next_character(mjb_character *character, mjb_character_ int mjbsh_character_command(int argc, char *const argv[], unsigned int flags) { mjb_status status = mjb_for_each_codepoint(argv[0], strlen(argv[0]), MJB_ENC_UTF_8, - mjbsh_output_next_character); + MJB_MALFORMED_STOP, mjbsh_output_next_character, NULL); if(status != MJB_STATUS_OK) { return mjbsh_error("%s", mjb_status_message(status)); diff --git a/src/shell/commands/emoji.c b/src/shell/commands/emoji.c index 533a8d03..450e9a8c 100644 --- a/src/shell/commands/emoji.c +++ b/src/shell/commands/emoji.c @@ -240,8 +240,8 @@ static void mjbsh_emoji_print_json(const char *buffer, size_t byte_length, bool mjbsh_emoji_json_character_count = 0; - if(mjb_for_each_codepoint(buffer, byte_length, MJB_ENC_UTF_8, mjbsh_emoji_next_character) != - MJB_STATUS_OK) { + if(mjb_for_each_codepoint(buffer, byte_length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + mjbsh_emoji_next_character, NULL) != MJB_STATUS_OK) { printf("]%s", mjbsh_jnl()); printf("}%s", mjbsh_jnl()); @@ -267,8 +267,8 @@ static void mjbsh_emoji_print_plain(const char *buffer, size_t byte_length, bool mjbsh_emoji_qualification_name(emoji->qualification), 1); mjbsh_numeric("Sequence Codepoints", 1, (unsigned int)emoji->codepoint_count); - if(mjb_for_each_codepoint(buffer, byte_length, MJB_ENC_UTF_8, mjbsh_emoji_next_character) != - MJB_STATUS_OK) { + if(mjb_for_each_codepoint(buffer, byte_length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + mjbsh_emoji_next_character, NULL) != MJB_STATUS_OK) { return; } } diff --git a/src/shell/commands/filter.c b/src/shell/commands/filter.c index 5c529859..6eb40006 100644 --- a/src/shell/commands/filter.c +++ b/src/shell/commands/filter.c @@ -12,8 +12,8 @@ static int mjbsh_print_filter_analysis(const char *input) { mjb_result result; size_t input_size = strlen(input); - mjb_status status = mjb_filter(input, input_size, MJB_ENC_UTF_8, filter_flags, MJB_ENC_UTF_8, - &result); + mjb_status status = mjb_filter(input, input_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + filter_flags, MJB_ENC_UTF_8, &result, NULL); if(status != MJB_STATUS_OK) { return mjbsh_error("%s", mjb_status_message(status)); diff --git a/src/shell/commands/normalize.c b/src/shell/commands/normalize.c index 7612db2e..9664fd28 100644 --- a/src/shell/commands/normalize.c +++ b/src/shell/commands/normalize.c @@ -10,8 +10,8 @@ int mjbsh_normalize_string_command(int argc, char *const argv[], unsigned int fl mjb_result result; bool ret = true; - mjb_status status = mjb_normalize(argv[0], strlen(argv[0]), MJB_ENC_UTF_8, - (mjb_normalization)flags, MJB_ENC_UTF_8, &result); + mjb_status status = mjb_normalize(argv[0], strlen(argv[0]), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + (mjb_normalization)flags, MJB_ENC_UTF_8, &result, NULL); if(status != MJB_STATUS_OK) { return mjbsh_error("%s", mjb_status_message(status)); @@ -25,8 +25,8 @@ int mjbsh_normalize_string_command(int argc, char *const argv[], unsigned int fl printf("%s", mjbsh_green()); if(result.output_size > 0 && - mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, - mjbsh_next_string_character) != MJB_STATUS_OK) { + mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + mjbsh_next_string_character, NULL) != MJB_STATUS_OK) { printf("%s", mjbsh_reset()); puts(""); ret = false; @@ -69,8 +69,8 @@ int mjbsh_normalize_command(int argc, char *const argv[], unsigned int flags) { mjb_result result; bool ret = true; - mjb_status status = mjb_normalize(codepoints, index, MJB_ENC_UTF_8, (mjb_normalization)flags, - MJB_ENC_UTF_8, &result); + mjb_status status = mjb_normalize(codepoints, index, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + (mjb_normalization)flags, MJB_ENC_UTF_8, &result, NULL); if(status != MJB_STATUS_OK) { free(codepoints); @@ -85,8 +85,8 @@ int mjbsh_normalize_command(int argc, char *const argv[], unsigned int flags) { } if(result.output_size > 0 && - mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, - mjbsh_next_codepoint) != MJB_STATUS_OK) { + mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + mjbsh_next_codepoint, NULL) != MJB_STATUS_OK) { puts(""); ret = false; diff --git a/src/shell/commands/string.c b/src/shell/commands/string.c index 8f9b635d..814a3f6b 100644 --- a/src/shell/commands/string.c +++ b/src/shell/commands/string.c @@ -8,8 +8,8 @@ int mjbsh_case_command(int argc, char *const argv[], unsigned int flags) { mjb_result result = { NULL, 0, false }; - mjb_status status = mjb_map_case(argv[0], strlen(argv[0]), MJB_ENC_UTF_8, - (mjb_map_case_type)flags, MJB_ENC_UTF_8, &result); + mjb_status status = mjb_map_case(argv[0], strlen(argv[0]), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + (mjb_map_case_type)flags, MJB_ENC_UTF_8, &result, NULL); if(status != MJB_STATUS_OK) { return mjbsh_error("%s", mjb_status_message(status)); diff --git a/src/shell/shell.c b/src/shell/shell.c index 03a3bacc..2f1456de 100644 --- a/src/shell/shell.c +++ b/src/shell/shell.c @@ -315,8 +315,8 @@ void mjbsh_normalization(const char *buffer_utf8, size_t utf8_length, mjb_normal bool is_json = cmd_output_mode == OUTPUT_MODE_JSON; mjb_result result; - bool ret = mjb_normalize(buffer_utf8, utf8_length, MJB_ENC_UTF_8, form, MJB_ENC_UTF_8, - &result) == MJB_STATUS_OK; + bool ret = mjb_normalize(buffer_utf8, utf8_length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, form, + MJB_ENC_UTF_8, &result, NULL) == MJB_STATUS_OK; if(ret) { if(is_json) { @@ -324,7 +324,7 @@ void mjbsh_normalization(const char *buffer_utf8, size_t utf8_length, mjb_normal cmd_json_indent == 0 ? "" : " ", mjbsh_green()); if(result.output_size > 0 && mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, - mjbsh_next_escaped_character) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, mjbsh_next_escaped_character, NULL) != MJB_STATUS_OK) { goto cleanup; } printf("%s\",%s", mjbsh_reset(), mjbsh_jnl()); @@ -345,8 +345,8 @@ void mjbsh_normalization(const char *buffer_utf8, size_t utf8_length, mjb_normal } if(result.output_size > 0 && - mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, - is_json ? mjbsh_next_array_codepoint : mjbsh_next_codepoint) != MJB_STATUS_OK) { + mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + is_json ? mjbsh_next_array_codepoint : mjbsh_next_codepoint, NULL) != MJB_STATUS_OK) { goto cleanup; } @@ -422,8 +422,8 @@ bool mjbsh_parse_codepoint(const char *input, mjb_codepoint *codepoint) { return true; } else { - if(mjb_for_each_codepoint(input, strlen(input), MJB_ENC_UTF_8, - mjbsh_next_current_character) != MJB_STATUS_OK) { + if(mjb_for_each_codepoint(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + mjbsh_next_current_character, NULL) != MJB_STATUS_OK) { return false; } diff --git a/src/string.c b/src/string.c index e7733429..88454280 100644 --- a/src/string.c +++ b/src/string.c @@ -178,12 +178,22 @@ mjb_status mjb_output_copy_into(const void *buffer, size_t byte_length, void *ou * Count the codepoints in a string. */ MJB_EXPORT mjb_status mjb_codepoint_count(const char *buffer, size_t byte_length, - mjb_encoding encoding, size_t *count) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, size_t *count, + mjb_diagnostic *diagnostic) { if(count == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } *count = 0; + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; + } + + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } if(byte_length == 0) { return MJB_STATUS_OK; @@ -193,32 +203,31 @@ MJB_EXPORT mjb_status mjb_codepoint_count(const char *buffer, size_t byte_length return MJB_STATUS_INVALID_ARGUMENT; } - if(!mjb_encoding_is_valid_input(encoding)) { - return MJB_STATUS_INVALID_ENCODING; - } - mjb_status status = mjb_resolve_input_byte_length(buffer, &byte_length, encoding); if(status != MJB_STATUS_OK || byte_length == 0) { return status; } - uint8_t state = MJB_UTF_ACCEPT; - bool in_error = false; mjb_codepoint codepoint = 0; size_t codepoint_count = 0; + size_t offset = 0; - for(size_t i = 0; i < byte_length;) { - mjb_decode_result decode_status = mjb_next_codepoint(buffer, byte_length, &state, &i, - encoding, &codepoint, &in_error); + for(;;) { + mjb_diagnostic current; + mjb_status decode_status = mjb_decode_next(buffer, byte_length, encoding, malformed_policy, + &offset, &codepoint, ¤t); + mjb_diagnostic_record(diagnostic, ¤t); - if(decode_status == MJB_DECODE_END) { + if(decode_status == MJB_STATUS_END_OF_INPUT) { break; } - if(decode_status == MJB_DECODE_OK || decode_status == MJB_DECODE_ERROR) { - ++codepoint_count; + if(decode_status != MJB_STATUS_OK) { + return decode_status; } + + ++codepoint_count; } *count = codepoint_count; diff --git a/src/terminal-width.c b/src/terminal-width.c index 2ace731b..24a7e8b2 100644 --- a/src/terminal-width.c +++ b/src/terminal-width.c @@ -86,17 +86,27 @@ static mjb_status mjb_terminal_cluster_width(const char *buffer, size_t byte_len * Return the estimated terminal-cell width of printable, single-line text. */ MJB_EXPORT mjb_status mjb_terminal_width(const char *buffer, size_t byte_length, - mjb_encoding encoding, mjb_terminal_width_profile profile, size_t *width) { + mjb_encoding encoding, mjb_malformed_policy malformed_policy, + mjb_terminal_width_profile profile, size_t *width, mjb_diagnostic *diagnostic) { if(width == NULL) { return MJB_STATUS_INVALID_ARGUMENT; } *width = 0; + mjb_diagnostic_reset(diagnostic); + + if(!mjb_malformed_policy_is_valid(malformed_policy)) { + return MJB_STATUS_INVALID_ARGUMENT; + } if(profile != MJB_TERMINAL_WIDTH_NARROW && profile != MJB_TERMINAL_WIDTH_EAST_ASIAN) { return MJB_STATUS_INVALID_ARGUMENT; } + if(!mjb_encoding_is_valid_input(encoding)) { + return MJB_STATUS_INVALID_ENCODING; + } + if(byte_length == 0) { return MJB_STATUS_OK; } @@ -112,8 +122,8 @@ MJB_EXPORT mjb_status mjb_terminal_width(const char *buffer, size_t byte_length, } mjb_result normalized = { NULL, 0, false }; - status = mjb_normalize(buffer, byte_length, encoding, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &normalized); + status = mjb_normalize(buffer, byte_length, encoding, malformed_policy, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_8, &normalized, diagnostic); if(status != MJB_STATUS_OK) { return status; diff --git a/src/ui/Mojibake/Mojibake/CaseView.swift b/src/ui/Mojibake/Mojibake/CaseView.swift index 81f382b0..2d6183ee 100644 --- a/src/ui/Mojibake/Mojibake/CaseView.swift +++ b/src/ui/Mojibake/Mojibake/CaseView.swift @@ -249,9 +249,11 @@ struct CaseView: View { input, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, type, MJB_ENC_UTF_8, - result + result, + nil ) } } diff --git a/src/ui/Mojibake/Mojibake/CharacterDetails.swift b/src/ui/Mojibake/Mojibake/CharacterDetails.swift index 35761187..729c2e5a 100644 --- a/src/ui/Mojibake/Mojibake/CharacterDetails.swift +++ b/src/ui/Mojibake/Mojibake/CharacterDetails.swift @@ -394,9 +394,11 @@ private struct CharacterDetailsBuilder { input, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, form, MJB_ENC_UTF_8, - result + result, + nil ) } } diff --git a/src/ui/Mojibake/Mojibake/CollationView.swift b/src/ui/Mojibake/Mojibake/CollationView.swift index e9c3c7a8..28c7d4cf 100644 --- a/src/ui/Mojibake/Mojibake/CollationView.swift +++ b/src/ui/Mojibake/Mojibake/CollationView.swift @@ -375,9 +375,11 @@ private struct CollationResolution { buffer, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, weighting, strength, - result + result, + nil ) return status } diff --git a/src/ui/Mojibake/Mojibake/EncodingInspectorView.swift b/src/ui/Mojibake/Mojibake/EncodingInspectorView.swift index e908a40a..e9c3cf10 100644 --- a/src/ui/Mojibake/Mojibake/EncodingInspectorView.swift +++ b/src/ui/Mojibake/Mojibake/EncodingInspectorView.swift @@ -426,7 +426,9 @@ private struct EncodingResolution { buffer.baseAddress, buffer.count, encoding, - &count + MJB_MALFORMED_STOP, + &count, + nil ) == MJB_STATUS_OK else { return 0 } @@ -460,8 +462,10 @@ private struct EncodingConversion: Identifiable { buffer, byteLength, inputEncoding, + MJB_MALFORMED_STOP, target.value, - result + result, + nil ) return status } diff --git a/src/ui/Mojibake/Mojibake/FilterView.swift b/src/ui/Mojibake/Mojibake/FilterView.swift index 3deb775c..3bc9b763 100644 --- a/src/ui/Mojibake/Mojibake/FilterView.swift +++ b/src/ui/Mojibake/Mojibake/FilterView.swift @@ -202,9 +202,11 @@ struct FilterView: View { input, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, filters, MJB_ENC_UTF_8, - result + result, + nil ) } } diff --git a/src/ui/Mojibake/Mojibake/MojibakeSupport.swift b/src/ui/Mojibake/Mojibake/MojibakeSupport.swift index 3aa855c8..afefbb92 100644 --- a/src/ui/Mojibake/Mojibake/MojibakeSupport.swift +++ b/src/ui/Mojibake/Mojibake/MojibakeSupport.swift @@ -109,7 +109,9 @@ enum MojibakeCounting { UnsafePointer?, Int, mjb_encoding, - UnsafeMutablePointer? + mjb_malformed_policy, + UnsafeMutablePointer?, + UnsafeMutablePointer? ) -> mjb_status ) -> Int? { let bytes = Array(value.utf8) @@ -122,7 +124,9 @@ enum MojibakeCounting { buffer.baseAddress, buffer.count, MJB_ENC_UTF_8, - &count + MJB_MALFORMED_STOP, + &count, + nil ) == MJB_STATUS_OK else { return nil } diff --git a/src/ui/Mojibake/Mojibake/NormalizationView.swift b/src/ui/Mojibake/Mojibake/NormalizationView.swift index 68ab9151..f3ef79b8 100644 --- a/src/ui/Mojibake/Mojibake/NormalizationView.swift +++ b/src/ui/Mojibake/Mojibake/NormalizationView.swift @@ -317,9 +317,11 @@ struct NormalizationView: View { input, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, form, MJB_ENC_UTF_8, - result + result, + nil ) } } @@ -330,8 +332,10 @@ struct NormalizationView: View { input, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, - result + result, + nil ) } } diff --git a/src/ui/Mojibake/Mojibake/TerminalWidthView.swift b/src/ui/Mojibake/Mojibake/TerminalWidthView.swift index 25428bc7..d162dfb9 100644 --- a/src/ui/Mojibake/Mojibake/TerminalWidthView.swift +++ b/src/ui/Mojibake/Mojibake/TerminalWidthView.swift @@ -346,8 +346,10 @@ private struct TerminalWidthAnalysis { buffer, byteLength, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, profile, - &width + &width, + nil ) guard status == MJB_STATUS_OK else { diff --git a/src/utf.h b/src/utf.h index 2c867554..7a12dd99 100644 --- a/src/utf.h +++ b/src/utf.h @@ -17,6 +17,64 @@ typedef enum { MJB_DECODE_ERROR // Invalid sequence (codepoint set to MJB_CODEPOINT_REPLACEMENT "�") } mjb_decode_result; +static inline bool MJB_USED mjb_malformed_policy_is_valid(mjb_malformed_policy policy) { + return policy == MJB_MALFORMED_STOP || policy == MJB_MALFORMED_REPLACE || + policy == MJB_MALFORMED_SKIP; +} + +static inline void MJB_USED mjb_diagnostic_reset(mjb_diagnostic *diagnostic) { + if(diagnostic == NULL) { + return; + } + + diagnostic->error = MJB_TEXT_ERROR_NONE; + diagnostic->byte_offset = 0; + diagnostic->byte_length = 0; + diagnostic->code_unit_offset = 0; +} + +static inline void MJB_USED mjb_diagnostic_record(mjb_diagnostic *destination, + const mjb_diagnostic *source) { + if(destination != NULL && source != NULL && destination->error == MJB_TEXT_ERROR_NONE && + source->error != MJB_TEXT_ERROR_NONE) { + *destination = *source; + } +} + +static inline void MJB_USED mjb_diagnose_malformed_input(const char *buffer, size_t byte_length, + mjb_encoding encoding, mjb_status status, mjb_diagnostic *diagnostic) { + if(status == MJB_STATUS_MALFORMED_INPUT && diagnostic != NULL && + diagnostic->error == MJB_TEXT_ERROR_NONE) { + (void)mjb_string_validate(buffer, byte_length, encoding, diagnostic); + } +} + +/** + * Repair input only after a strict traversal has found malformed text. Valid input therefore pays + * no separate validation cost. + */ +static inline mjb_status MJB_USED mjb_repair_text_input(const char **buffer, size_t *byte_length, + mjb_encoding *encoding, mjb_malformed_policy malformed_policy, mjb_diagnostic *diagnostic, + mjb_result *sanitized) { + if(malformed_policy == MJB_MALFORMED_STOP) { + mjb_diagnose_malformed_input(*buffer, *byte_length, *encoding, MJB_STATUS_MALFORMED_INPUT, + diagnostic); + + return MJB_STATUS_MALFORMED_INPUT; + } + + mjb_status status = mjb_convert_encoding(*buffer, *byte_length, *encoding, malformed_policy, + MJB_ENC_UTF_8, sanitized, diagnostic); + + if(status == MJB_STATUS_OK) { + *buffer = sanitized->output; + *byte_length = sanitized->output_size; + *encoding = MJB_ENC_UTF_8; + } + + return status; +} + static inline bool MJB_USED mjb_utf_state_is_incomplete(uint8_t state) { return state != MJB_UTF_ACCEPT && state != MJB_UTF_REJECT; } @@ -105,9 +163,26 @@ static inline mjb_encoding MJB_USED mjb_resolve_input_encoding(const char *buffe return encoding; } +static inline mjb_status MJB_USED mjb_check_input_encoding_byte_order(const char *buffer, + size_t byte_length, mjb_encoding encoding) { + size_t index = 0; + mjb_encoding resolved = mjb_resolve_input_encoding(buffer, byte_length, encoding, &index); + + if((encoding == MJB_ENC_UTF_16 || encoding == MJB_ENC_UTF_32) && resolved == encoding) { + return MJB_STATUS_INVALID_ENCODING; + } + + return MJB_STATUS_OK; +} + static inline bool MJB_USED mjb_decode_step(const char *buffer, size_t byte_length, uint8_t *state, size_t *index, mjb_encoding encoding, mjb_codepoint *codepoint) { - if(encoding == MJB_ENC_UTF_8 || encoding == MJB_ENC_ASCII) { + if(encoding == MJB_ENC_ASCII) { + uint8_t byte = (uint8_t)buffer[*index]; + *codepoint = byte; + *state = byte <= 0x7F ? MJB_UTF_ACCEPT : MJB_UTF_REJECT; + ++*index; + } else if(encoding == MJB_ENC_UTF_8) { *state = mjb_utf8_decode_step(*state, buffer[*index], codepoint); ++*index; // Increment by 1 byte } else if(encoding == MJB_ENC_UTF_16BE || encoding == MJB_ENC_UTF_16LE) { @@ -374,49 +449,3 @@ static inline mjb_status MJB_USED mjb_resolve_input_byte_length(const char *buff index += code_unit_size; } } - -/** - * Validate a complete code-unit sequence without producing output. - */ -static inline mjb_status MJB_USED mjb_validate_code_unit_sequence(const char *buffer, - size_t byte_length, mjb_encoding encoding) { - if(!mjb_encoding_is_valid_input(encoding)) { - return MJB_STATUS_INVALID_ENCODING; - } - - if(byte_length == 0) { - return MJB_STATUS_OK; - } - - size_t resolved_index = 0; - mjb_encoding resolved_encoding = mjb_resolve_input_encoding(buffer, byte_length, encoding, - &resolved_index); - - if((encoding == MJB_ENC_UTF_16 || encoding == MJB_ENC_UTF_32) && - resolved_encoding == encoding) { - return MJB_STATUS_INVALID_ENCODING; - } - - uint8_t state = MJB_UTF_ACCEPT; - bool in_error = false; - mjb_codepoint codepoint = 0; - - for(size_t i = 0; i < byte_length;) { - mjb_decode_result result = mjb_next_codepoint(buffer, byte_length, &state, &i, encoding, - &codepoint, &in_error); - - if(result == MJB_DECODE_END) { - break; - } - - if(result == MJB_DECODE_ERROR) { - return MJB_STATUS_MALFORMED_INPUT; - } - } - - if(mjb_utf_state_is_incomplete(state)) { - return MJB_STATUS_MALFORMED_INPUT; - } - - return MJB_STATUS_OK; -} diff --git a/tests/bidi.c b/tests/bidi.c index 7215de03..be10da0f 100644 --- a/tests/bidi.c +++ b/tests/bidi.c @@ -243,6 +243,11 @@ int test_bidi(void *arg) { ATT_ASSERT(para.count, (size_t)0, "empty string count") mjb_bidi_paragraph_free(¶); + status = mjb_bidi_resolve("A\x80", 2, MJB_ENC_UTF_8, MJB_DIRECTION_AUTO, ¶); + ATT_ASSERT_STATUS(status, MJB_STATUS_MALFORMED_INPUT, "resolve rejects malformed input") + ATT_ASSERT(para.chars, (mjb_bidi_char *)NULL, "malformed resolve leaves no characters") + ATT_ASSERT(para.count, (size_t)0, "malformed resolve leaves a zero count") + const char *ltr = "ABC"; status = mjb_bidi_resolve(ltr, strlen(ltr), MJB_ENC_UTF_8, MJB_DIRECTION_AUTO, ¶); ATT_ASSERT_STATUS(status, MJB_STATUS_OK, "LTR resolve ok") diff --git a/tests/break-sentence.c b/tests/break-sentence.c index cfaa1e86..dc730cc9 100644 --- a/tests/break-sentence.c +++ b/tests/break-sentence.c @@ -42,60 +42,79 @@ static void test_sentence_count(void) { size_t count = 6251; // Argument validation - ATT_ASSERT_STATUS(mjb_sentence_count("A", 1, MJB_ENC_UTF_8, NULL), + ATT_ASSERT_STATUS(mjb_sentence_count("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Count rejects NULL count") - ATT_ASSERT_STATUS(mjb_sentence_count(NULL, 1, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_INVALID_ARGUMENT, "Count rejects NULL buffer") ATT_ASSERT(count, (size_t)0, "Count is zero after NULL buffer") count = 6251; - ATT_ASSERT_STATUS(mjb_sentence_count("A", 1, MJB_ENC_UNKNOWN, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("A", 1, MJB_ENC_UNKNOWN, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_INVALID_ENCODING, "Count rejects invalid encoding") ATT_ASSERT(count, (size_t)0, "Count is zero after invalid encoding") // Empty input is valid and counts zero segments - ATT_ASSERT_STATUS(mjb_sentence_count("", 0, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_sentence_count("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: empty string status") ATT_ASSERT(count, (size_t)0, "Count: empty string") - ATT_ASSERT_STATUS(mjb_sentence_count(NULL, 0, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_sentence_count(NULL, 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: NULL buffer with zero length status") + mjb_diagnostic diagnostic; + ATT_ASSERT_STATUS(mjb_sentence_count("A\x80", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Count rejects malformed input") + ATT_ASSERT(count, (size_t)0, "Count is zero after malformed input") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Sentence count malformed offset") + + ATT_ASSERT_STATUS(mjb_sentence_count("A\x80", 2, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &count, &diagnostic), + MJB_STATUS_OK, "Sentence count replaces malformed input") + ATT_ASSERT(count, (size_t)1, "Replacement remains in one sentence") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, + "Sentence replacement retains malformed offset") + + ATT_ASSERT_STATUS(mjb_sentence_count("A\x80", 2, MJB_ENC_UTF_8, MJB_MALFORMED_SKIP, + &count, &diagnostic), + MJB_STATUS_OK, "Sentence count skips malformed input") + ATT_ASSERT(count, (size_t)1, "Skipping malformed input remains one sentence") + // Text without a terminator is a single segment - ATT_ASSERT_STATUS(mjb_sentence_count("One sentence", 12, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("One sentence", 12, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: one sentence status") ATT_ASSERT(count, (size_t)1, "Count: one sentence") // Terminators split segments (SB4, SB11) - ATT_ASSERT_STATUS(mjb_sentence_count("Hello. How are you? Fine!", 25, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("Hello. How are you? Fine!", 25, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: three sentences status") ATT_ASSERT(count, (size_t)3, "Count: three sentences") - ATT_ASSERT_STATUS(mjb_sentence_count("One.\nTwo.\n", 10, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("One.\nTwo.\n", 10, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: newline paragraphs status") ATT_ASSERT(count, (size_t)2, "Count: newline paragraphs") // Trailing spaces attach to the previous sentence (SB10) - ATT_ASSERT_STATUS(mjb_sentence_count("Hi. ", 6, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_sentence_count("Hi. ", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: trailing spaces status") ATT_ASSERT(count, (size_t)1, "Count: trailing spaces") // No break when the terminator is followed by lowercase (SB8) - ATT_ASSERT_STATUS(mjb_sentence_count("Wait... then more.", 18, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("Wait... then more.", 18, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: ellipsis status") ATT_ASSERT(count, (size_t)1, "Count: ellipsis before lowercase") // Default rules carry no abbreviation list - ATT_ASSERT_STATUS(mjb_sentence_count("Dr. Smith went.", 15, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("Dr. Smith went.", 15, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: abbreviation status") ATT_ASSERT(count, (size_t)2, "Count: abbreviation splits by default") // MJB_NUL_TERMINATED requests a terminator scan - ATT_ASSERT_STATUS(mjb_sentence_count("Hi. Bye.", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_sentence_count("Hi. Bye.", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: NUL-terminated status") ATT_ASSERT(count, (size_t)2, "Count: NUL-terminated") // UTF-16LE input const char utf16le_two[] = { 'A', '\0', '.', '\0', ' ', '\0', 'B', '\0', '.', '\0' }; - ATT_ASSERT_STATUS(mjb_sentence_count(utf16le_two, 10, MJB_ENC_UTF_16LE, &count), + ATT_ASSERT_STATUS(mjb_sentence_count(utf16le_two, 10, MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: UTF-16LE status") ATT_ASSERT(count, (size_t)2, "Count: UTF-16LE two sentences") } diff --git a/tests/break-word.c b/tests/break-word.c index dd6681f6..49e8ee45 100644 --- a/tests/break-word.c +++ b/tests/break-word.c @@ -94,84 +94,104 @@ static void test_word_count(void) { size_t count = 6251; // Argument validation - ATT_ASSERT_STATUS(mjb_word_count("A", 1, MJB_ENC_UTF_8, NULL), + ATT_ASSERT_STATUS(mjb_word_count("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Count rejects NULL count") - ATT_ASSERT_STATUS(mjb_word_count(NULL, 1, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_INVALID_ARGUMENT, "Count rejects NULL buffer") ATT_ASSERT(count, (size_t)0, "Count is zero after NULL buffer") count = 6251; - ATT_ASSERT_STATUS(mjb_word_count("A", 1, MJB_ENC_UNKNOWN, &count), + ATT_ASSERT_STATUS(mjb_word_count("A", 1, MJB_ENC_UNKNOWN, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_INVALID_ENCODING, "Count rejects invalid encoding") ATT_ASSERT(count, (size_t)0, "Count is zero after invalid encoding") // Empty input is valid and counts zero words - ATT_ASSERT_STATUS(mjb_word_count("", 0, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: empty string status") ATT_ASSERT(count, (size_t)0, "Count: empty string") - ATT_ASSERT_STATUS(mjb_word_count(NULL, 0, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count(NULL, 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: NULL buffer with zero length status") + mjb_diagnostic diagnostic; + ATT_ASSERT_STATUS(mjb_word_count("A\x80", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, + &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Count rejects malformed input") + ATT_ASSERT(count, (size_t)0, "Count is zero after malformed input") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Word count malformed offset") + + const char malformed_word[] = "A\x80" + "B"; + ATT_ASSERT_STATUS(mjb_word_count(malformed_word, sizeof(malformed_word) - 1, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &count, &diagnostic), + MJB_STATUS_OK, "Word count replaces malformed input") + ATT_ASSERT(count, (size_t)2, "Replacement separates word segments") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Word replacement retains malformed offset") + + ATT_ASSERT_STATUS(mjb_word_count(malformed_word, sizeof(malformed_word) - 1, MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, &count, &diagnostic), + MJB_STATUS_OK, "Word count skips malformed input") + ATT_ASSERT(count, (size_t)1, "Skipping malformed input joins adjacent letters") + // Punctuation, whitespace, and symbol segments are not counted - ATT_ASSERT_STATUS(mjb_word_count("Hello, world! It works.", 23, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count("Hello, world! It works.", 23, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: sentence status") ATT_ASSERT(count, (size_t)4, "Count: sentence has four words") - ATT_ASSERT_STATUS(mjb_word_count("...", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count("...", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: punctuation-only status") ATT_ASSERT(count, (size_t)0, "Count: punctuation-only") - ATT_ASSERT_STATUS(mjb_word_count(" ", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count(" ", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: spaces-only status") ATT_ASSERT(count, (size_t)0, "Count: spaces-only") // WB6/WB7 keep letters together across certain punctuation - ATT_ASSERT_STATUS(mjb_word_count("don't stop", 10, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count("don't stop", 10, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: apostrophe status") ATT_ASSERT(count, (size_t)2, "Count: apostrophe stays inside the word") - ATT_ASSERT_STATUS(mjb_word_count("e.g. example.com", 16, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count("e.g. example.com", 16, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: abbreviation status") ATT_ASSERT(count, (size_t)2, "Count: mid-letter punctuation stays inside the word") // Hyphenated compounds count each part (WB999 breaks at the hyphens) - ATT_ASSERT_STATUS(mjb_word_count("state-of-the-art", 16, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count("state-of-the-art", 16, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: hyphenated status") ATT_ASSERT(count, (size_t)4, "Count: hyphenated compound counts each part") // Numbers are word-like (WB11/WB12 keep formatted numbers together) - ATT_ASSERT_STATUS(mjb_word_count("3.14 is pi", 10, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count("3.14 is pi", 10, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: number status") ATT_ASSERT(count, (size_t)3, "Count: formatted number is one word") // ExtendNumLet joins words; a connector alone is not a word (WB13a, WB13b) - ATT_ASSERT_STATUS(mjb_word_count("foo_bar baz", 11, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count("foo_bar baz", 11, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: connector status") ATT_ASSERT(count, (size_t)2, "Count: connector joins one word") - ATT_ASSERT_STATUS(mjb_word_count("_ _", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count("_ _", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: connector-only status") ATT_ASSERT(count, (size_t)0, "Count: connector-only segments are not words") // Emoji are not word-like - ATT_ASSERT_STATUS(mjb_word_count("a \xF0\x9F\x91\x8D b", 8, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count("a \xF0\x9F\x91\x8D b", 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: emoji status") ATT_ASSERT(count, (size_t)2, "Count: emoji between words") // No dictionary segmentation: ideographs count one word per character - ATT_ASSERT_STATUS(mjb_word_count("\xE6\x9D\xB1\xE4\xBA\xAC", 6, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count("\xE6\x9D\xB1\xE4\xBA\xAC", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: ideograph status") ATT_ASSERT(count, (size_t)2, "Count: one word per ideograph") // Katakana runs stay together (WB13) ATT_ASSERT_STATUS(mjb_word_count("\xE3\x82\xAB\xE3\x82\xBF\xE3\x82\xAB\xE3\x83\x8A", 12, - MJB_ENC_UTF_8, &count), MJB_STATUS_OK, "Count: katakana status") + MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: katakana status") ATT_ASSERT(count, (size_t)1, "Count: katakana run is one word") // MJB_NUL_TERMINATED requests a terminator scan - ATT_ASSERT_STATUS(mjb_word_count("Hi there", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_word_count("Hi there", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: NUL-terminated status") ATT_ASSERT(count, (size_t)2, "Count: NUL-terminated") // UTF-16LE input const char utf16le_two[] = { 'H', '\0', 'i', '\0', ' ', '\0', 'y', '\0', 'o', '\0', 'u', '\0' }; - ATT_ASSERT_STATUS(mjb_word_count(utf16le_two, 12, MJB_ENC_UTF_16LE, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_word_count(utf16le_two, 12, MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: UTF-16LE status") ATT_ASSERT(count, (size_t)2, "Count: UTF-16LE two words") } diff --git a/tests/case.c b/tests/case.c index fe377195..451edee5 100644 --- a/tests/case.c +++ b/tests/case.c @@ -126,23 +126,60 @@ int test_case(void *arg) { MJB_TEST_COVERAGE(mjb_map_case_into); size_t into_size = 11; - ATT_ASSERT_STATUS(mjb_map_case_into(NULL, 1, encoding, MJB_CASE_UPPER, encoding, NULL, - &into_size), + ATT_ASSERT_STATUS(mjb_map_case_into(NULL, 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, encoding, NULL, + &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Case into rejects NULL buffer") ATT_ASSERT(into_size, (size_t)0, "Case into resets size for invalid input") - ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, encoding, MJB_CASE_UPPER, encoding, NULL, NULL), + ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, encoding, NULL, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Case into rejects NULL output size") - ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, encoding, MJB_CASE_NONE, encoding, NULL, - &into_size), + ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_NONE, encoding, NULL, + &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Case into rejects MJB_CASE_NONE") + into_size = 11; + ATT_ASSERT_STATUS(mjb_map_case_into("A\x80", 2, encoding, MJB_MALFORMED_STOP, MJB_CASE_LOWER, encoding, NULL, + &into_size, NULL), + MJB_STATUS_MALFORMED_INPUT, "Case into rejects malformed input") + ATT_ASSERT(into_size, (size_t)0, "Malformed case input resets output size") + ATT_ASSERT_STATUS(mjb_map_case("A\x80", 2, encoding, MJB_MALFORMED_STOP, MJB_CASE_LOWER, encoding, + &guard_result, NULL), + MJB_STATUS_MALFORMED_INPUT, "Case mapping rejects malformed input") + + const char malformed_case[] = "a\x80" + "b"; + mjb_diagnostic diagnostic; + ATT_ASSERT_STATUS(mjb_map_case(malformed_case, sizeof(malformed_case) - 1, encoding, + MJB_MALFORMED_REPLACE, MJB_CASE_UPPER, encoding, &guard_result, + &diagnostic), + MJB_STATUS_OK, "Case mapping replaces malformed input") + ATT_ASSERT(guard_result.output_size, (size_t)5, "Case replacement output size") + ATT_ASSERT((int)memcmp(guard_result.output, "A\xEF\xBF\xBD" + "B", + guard_result.output_size), + 0, "Case replacement output") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Case replacement diagnostic offset") + ATT_ASSERT_STATUS(mjb_result_free(&guard_result), MJB_STATUS_OK, + "Free case replacement result") + + ATT_ASSERT_STATUS(mjb_map_case(malformed_case, sizeof(malformed_case) - 1, encoding, + MJB_MALFORMED_SKIP, MJB_CASE_UPPER, encoding, &guard_result, + &diagnostic), + MJB_STATUS_OK, "Case mapping skips malformed input") + ATT_ASSERT(guard_result.output_size, (size_t)2, "Case skip output size") + ATT_ASSERT((int)memcmp(guard_result.output, "AB", guard_result.output_size), 0, + "Case skip output") + ATT_ASSERT_STATUS(mjb_result_free(&guard_result), MJB_STATUS_OK, "Free case skip result") + + ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, (mjb_malformed_policy)99, + MJB_CASE_UPPER, encoding, &guard_result, NULL), + MJB_STATUS_INVALID_ARGUMENT, "Case mapping rejects invalid malformed policy") const char *into_input = "Stra\xC3\x9F" "e"; const char *into_expected = "STRASSE"; into_size = 0; - ATT_ASSERT_STATUS(mjb_map_case_into(into_input, strlen(into_input), encoding, MJB_CASE_UPPER, - encoding, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_map_case_into(into_input, strlen(into_input), encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, + encoding, NULL, &into_size, NULL), MJB_STATUS_OK, "Query case-mapped output size") ATT_ASSERT(into_size, strlen(into_expected), "Case into required payload size") @@ -152,16 +189,16 @@ int test_case(void *arg) { memset(untouched_output, 0xA5, sizeof(untouched_output)); into_size = strlen(into_expected) - 1; - ATT_ASSERT_STATUS(mjb_map_case_into(into_input, strlen(into_input), encoding, MJB_CASE_UPPER, - encoding, into_output, &into_size), + ATT_ASSERT_STATUS(mjb_map_case_into(into_input, strlen(into_input), encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, + encoding, into_output, &into_size, NULL), MJB_STATUS_OUTPUT_TOO_SMALL, "Case into reports a small output buffer") ATT_ASSERT(into_size, strlen(into_expected), "Small case output reports required size") ATT_ASSERT(memcmp(into_output, untouched_output, sizeof(into_output)), 0, "Small case output buffer is not modified") into_size = strlen(into_expected); - ATT_ASSERT_STATUS(mjb_map_case_into(into_input, strlen(into_input), encoding, MJB_CASE_UPPER, - encoding, into_output, &into_size), + ATT_ASSERT_STATUS(mjb_map_case_into(into_input, strlen(into_input), encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, + encoding, into_output, &into_size, NULL), MJB_STATUS_OK, "Case into exact-size output buffer") ATT_ASSERT(into_size, strlen(into_expected), "Case into written payload size") ATT_ASSERT(memcmp(into_output, into_expected, into_size), 0, "Case into output bytes") @@ -169,20 +206,20 @@ int test_case(void *arg) { "Case into does not write a terminator") into_size = sizeof(into_output); - ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, encoding, MJB_CASE_UPPER, MJB_ENC_UTF_16LE, - into_output, &into_size), + ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_16LE, + into_output, &into_size, NULL), MJB_STATUS_OK, "Case into converts output encoding") ATT_ASSERT(into_size, (size_t)2, "Case into converted output size") ATT_ASSERT(memcmp(into_output, "A\0", 2), 0, "Case into converted output bytes") - ATT_ASSERT_STATUS(mjb_map_case(NULL, 1, encoding, MJB_CASE_UPPER, encoding, &guard_result), + ATT_ASSERT_STATUS(mjb_map_case(NULL, 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, encoding, &guard_result, NULL), MJB_STATUS_INVALID_ARGUMENT, "Case conversion rejects NULL buffer") - ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, MJB_CASE_UPPER, encoding, NULL), + ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, encoding, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Case conversion rejects NULL result") - ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, MJB_CASE_NONE, encoding, &guard_result), + ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_NONE, encoding, &guard_result, NULL), MJB_STATUS_INVALID_ARGUMENT, "Case conversion rejects MJB_CASE_NONE") - ATT_ASSERT_STATUS(mjb_map_case("", 0, encoding, MJB_CASE_UPPER, encoding, &guard_result), + ATT_ASSERT_STATUS(mjb_map_case("", 0, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, encoding, &guard_result, NULL), MJB_STATUS_OK, "Case conversion accepts empty string") ATT_ASSERT(guard_result.transformed, false, "Case conversion empty string not transformed") ATT_ASSERT(guard_result.output_size, (size_t)0, "Case conversion empty string size") @@ -195,7 +232,7 @@ int test_case(void *arg) { ATT_ASSERT(result, "\xE1\x8E\xA0", "Lowercase Cherokee casefolds to uppercase") mjb_test_free(result); - ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, MJB_CASE_UPPER, MJB_ENC_UTF_16LE, &guard_result), + ATT_ASSERT_STATUS(mjb_map_case("a", 1, encoding, MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_16LE, &guard_result, NULL), MJB_STATUS_OK, "Case conversion converts output encoding") ATT_ASSERT(guard_result.transformed, true, "Case conversion converted output encoding transformed") diff --git a/tests/collation.c b/tests/collation.c index b81e5a89..52e3d208 100644 --- a/tests/collation.c +++ b/tests/collation.c @@ -139,10 +139,10 @@ static int compare_shifted_test_strings(const char *left, size_t left_len, const mjb_result left_key = { NULL, 0, false }; mjb_result right_key = { NULL, 0, false }; - if(mjb_collation_key(left, left_len, MJB_ENC_UTF_8, MJB_COLLATION_SHIFTED, - MJB_COLLATION_QUATERNARY, &left_key) != MJB_STATUS_OK || - mjb_collation_key(right, right_len, MJB_ENC_UTF_8, MJB_COLLATION_SHIFTED, - MJB_COLLATION_QUATERNARY, &right_key) != MJB_STATUS_OK) { + if(mjb_collation_key(left, left_len, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_SHIFTED, + MJB_COLLATION_QUATERNARY, &left_key, NULL) != MJB_STATUS_OK || + mjb_collation_key(right, right_len, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_SHIFTED, MJB_COLLATION_QUATERNARY, &right_key, NULL) != MJB_STATUS_OK) { mjb_result_free(&left_key); mjb_result_free(&right_key); @@ -186,8 +186,8 @@ static void assert_collation_primaries(mjb_codepoint codepoint, uint16_t first, size_t length = mjb_codepoint_encode(codepoint, utf8, sizeof(utf8), MJB_ENC_UTF_8); mjb_result key = { NULL, 0, false }; - ATT_ASSERT_STATUS(mjb_collation_key(utf8, length, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_PRIMARY, &key), + ATT_ASSERT_STATUS(mjb_collation_key(utf8, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_PRIMARY, &key, NULL), MJB_STATUS_OK, message) ATT_ASSERT((int)(key.output_size >= 4), 1, message) ATT_ASSERT((int)sort_key_word(&key, 0), (int)first, message) @@ -202,8 +202,8 @@ static void assert_collation_malformed_utf8(const unsigned char *buffer, size_t const char *bytes = (const char *)buffer; int order = 42; - ATT_ASSERT_STATUS(mjb_collation_key(bytes, byte_length, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &result), + ATT_ASSERT_STATUS(mjb_collation_key(bytes, byte_length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &result, NULL), MJB_STATUS_MALFORMED_INPUT, message) ATT_ASSERT_STATUS(mjb_collation_compare(bytes, byte_length, MJB_ENC_UTF_8, bytes, byte_length, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, @@ -304,25 +304,25 @@ int test_collation(void *arg) { // mjb_collation_key mjb_result ka, kb, kc, kd; - ATT_ASSERT_STATUS(mjb_collation_key(NULL, 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &ka, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key rejects NULL buffer") - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, NULL), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key rejects NULL result") size_t into_size = 9; - ATT_ASSERT_STATUS(mjb_collation_key_into(NULL, 1, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_collation_key_into(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key into rejects NULL buffer") ATT_ASSERT(into_size, (size_t)0, "Key into clears size after invalid input") - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, NULL), + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key into rejects NULL size") into_size = 9; - ATT_ASSERT_STATUS(mjb_collation_key_into("", 0, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_collation_key_into("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &into_size, NULL), MJB_STATUS_OK, "Key into measures empty input") ATT_ASSERT(into_size, (size_t)0, "Key into empty size") int order = 42; @@ -345,15 +345,15 @@ int test_collation(void *arg) { ATT_ASSERT_STATUS(mjb_collation_compare("a", 1, MJB_ENC_UTF_8, "b", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, (mjb_collation_strength)99, &order), MJB_STATUS_INVALID_ARGUMENT, "Compare rejects invalid strength") - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, - (mjb_collation_variable_weighting)99, MJB_COLLATION_TERTIARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + (mjb_collation_variable_weighting)99, MJB_COLLATION_TERTIARY, &ka, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key rejects invalid variable weighting") - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - (mjb_collation_strength)99, &ka), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + (mjb_collation_strength)99, &ka, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key rejects invalid strength") into_size = 9; - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, (mjb_collation_strength)99, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, (mjb_collation_strength)99, NULL, &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Key into rejects invalid strength") ATT_ASSERT(into_size, (size_t)0, "Key into clears size after invalid strength") @@ -377,18 +377,61 @@ int test_collation(void *arg) { assert_collation_malformed_utf8(invalid_four_byte, sizeof(invalid_four_byte), "Collation rejects invalid four-byte UTF-8"); + const char malformed_key_input[] = "a\x80" + "b"; + mjb_diagnostic diagnostic; + mjb_result recovered_key = { NULL, 0, false }; + mjb_result expected_key = { NULL, 0, false }; + ATT_ASSERT_STATUS(mjb_collation_key(malformed_key_input, sizeof(malformed_key_input) - 1, + MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &recovered_key, + &diagnostic), + MJB_STATUS_OK, "Collation key replaces malformed input") + ATT_ASSERT_STATUS(mjb_collation_key("a\xEF\xBF\xBD" + "b", + 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &expected_key, + NULL), + MJB_STATUS_OK, "Collation key for explicit replacement") + ATT_ASSERT(recovered_key.output_size, expected_key.output_size, + "Replacement collation key size") + ATT_ASSERT((int)memcmp(recovered_key.output, expected_key.output, recovered_key.output_size), + 0, "Replacement collation key bytes") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, + "Collation replacement retains malformed offset") + ATT_ASSERT_STATUS(mjb_result_free(&recovered_key), MJB_STATUS_OK, + "Free recovered collation key") + ATT_ASSERT_STATUS(mjb_result_free(&expected_key), MJB_STATUS_OK, + "Free expected replacement collation key") + + ATT_ASSERT_STATUS(mjb_collation_key(malformed_key_input, sizeof(malformed_key_input) - 1, + MJB_ENC_UTF_8, MJB_MALFORMED_SKIP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &recovered_key, &diagnostic), + MJB_STATUS_OK, "Collation key skips malformed input") + ATT_ASSERT_STATUS(mjb_collation_key("ab", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &expected_key, + NULL), + MJB_STATUS_OK, "Collation key without malformed input") + ATT_ASSERT(recovered_key.output_size, expected_key.output_size, "Skip collation key size") + ATT_ASSERT((int)memcmp(recovered_key.output, expected_key.output, recovered_key.output_size), + 0, "Skip collation key bytes") + ATT_ASSERT_STATUS(mjb_result_free(&recovered_key), MJB_STATUS_OK, + "Free skipped collation key") + ATT_ASSERT_STATUS(mjb_result_free(&expected_key), MJB_STATUS_OK, + "Free expected skipped collation key") + // Key generation succeeds - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &ka, NULL), MJB_STATUS_OK, "Key: 'a' succeeds") - ATT_ASSERT_STATUS(mjb_collation_key("b", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &kb), + ATT_ASSERT_STATUS(mjb_collation_key("b", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &kb, NULL), MJB_STATUS_OK, "Key: 'b' succeeds") MJB_TEST_COVERAGE(mjb_collation_key_into); into_size = 0; - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &into_size, NULL), MJB_STATUS_OK, "Key into queries required size") ATT_ASSERT(into_size, ka.output_size, "Key into required size") @@ -398,9 +441,9 @@ int test_collation(void *arg) { if(into_size > 0 && into_size < sizeof(into_output)) { memset(into_output, 0xA5, sizeof(into_output)); size_t small_size = into_size - 1; - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, into_output, - &small_size), + &small_size, NULL), MJB_STATUS_OUTPUT_TOO_SMALL, "Key into reports a small output buffer") ATT_ASSERT(small_size, into_size, "Key into preserves required size") @@ -410,9 +453,9 @@ int test_collation(void *arg) { "Key into leaves a small buffer untouched") size_t exact_size = into_size; - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, into_output, - &exact_size), + &exact_size, NULL), MJB_STATUS_OK, "Key into writes into exact capacity") ATT_ASSERT(exact_size, ka.output_size, "Key into written size") ATT_ASSERT((int)memcmp(into_output, ka.output, exact_size), 0, "Key into output") @@ -428,10 +471,10 @@ int test_collation(void *arg) { 1, "Key: 'a' < 'b'") // Same string -> identical keys - ATT_ASSERT_STATUS(mjb_collation_key("hello", 5, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &kc), MJB_STATUS_OK, "Key: 'hello' NI succeeds") - ATT_ASSERT_STATUS(mjb_collation_key("hello", 5, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &kd), MJB_STATUS_OK, "Key: 'hello' NI second succeeds") + ATT_ASSERT_STATUS(mjb_collation_key("hello", 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &kc, NULL), MJB_STATUS_OK, "Key: 'hello' NI succeeds") + ATT_ASSERT_STATUS(mjb_collation_key("hello", 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &kd, NULL), MJB_STATUS_OK, "Key: 'hello' NI second succeeds") ATT_ASSERT((int)(kc.output_size == kd.output_size), 1, "Key: same size") ATT_ASSERT((int)(memcmp(kc.output, kd.output, kc.output_size) == 0), 1, "Key: 'hello' == 'hello'") @@ -440,11 +483,11 @@ int test_collation(void *arg) { mjb_result_free(&kd); // NON_IGNORABLE vs SHIFTED differ for strings with variable-weight punctuation - ATT_ASSERT_STATUS(mjb_collation_key("a-b", 3, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &kc), + ATT_ASSERT_STATUS(mjb_collation_key("a-b", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &kc, NULL), MJB_STATUS_OK, "Key: 'a-b' NI succeeds") - ATT_ASSERT_STATUS(mjb_collation_key("a-b", 3, MJB_ENC_UTF_8, MJB_COLLATION_SHIFTED, - MJB_COLLATION_QUATERNARY, &kd), + ATT_ASSERT_STATUS(mjb_collation_key("a-b", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_SHIFTED, + MJB_COLLATION_QUATERNARY, &kd, NULL), MJB_STATUS_OK, "Key: 'a-b' SHIFTED succeeds") ATT_ASSERT((int)(kc.output_size != kd.output_size || memcmp(kc.output, kd.output, kc.output_size) != 0), 1, "Key: NI != SHIFTED for 'a-b'") @@ -463,10 +506,10 @@ int test_collation(void *arg) { assert_collation_primaries(0xFFFF, 0xFFFF, 0x0000, "U+FFFF reserved high weight"); // Key ordering matches mjb_collation_compare - ATT_ASSERT_STATUS(mjb_collation_key("apple", 5, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &ka), MJB_STATUS_OK, "Key: 'apple' succeeds") - ATT_ASSERT_STATUS(mjb_collation_key("banana", 6, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &kb), MJB_STATUS_OK, "Key: 'banana' succeeds") + ATT_ASSERT_STATUS(mjb_collation_key("apple", 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &ka, NULL), MJB_STATUS_OK, "Key: 'apple' succeeds") + ATT_ASSERT_STATUS(mjb_collation_key("banana", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &kb, NULL), MJB_STATUS_OK, "Key: 'banana' succeeds") int cmp_direct = test_collation_compare("apple", 5, MJB_ENC_UTF_8, "banana", 6, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY); @@ -516,25 +559,25 @@ int test_collation(void *arg) { MJB_COLLATION_SHIFTED, MJB_COLLATION_QUATERNARY) != 0), 1, "Shifted punctuation differs from empty at quaternary strength") - ATT_ASSERT_STATUS(mjb_collation_key("", 0, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &ka, NULL), MJB_STATUS_OK, "Empty tertiary key succeeds") - ATT_ASSERT_STATUS(mjb_collation_key(zero_width_space, 3, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &kb), + ATT_ASSERT_STATUS(mjb_collation_key(zero_width_space, 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &kb, NULL), MJB_STATUS_OK, "Completely ignorable tertiary key succeeds") ATT_ASSERT(ka.output_size, (size_t)0, "Empty tertiary key has no effective weights") ATT_ASSERT(kb.output_size, (size_t)0, "Completely ignorable tertiary key is empty") mjb_result_free(&ka); mjb_result_free(&kb); - ATT_ASSERT_STATUS(mjb_collation_key(combining_acute, 2, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_PRIMARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key(combining_acute, 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_PRIMARY, &ka, NULL), MJB_STATUS_OK, "Primary-ignorable accent key succeeds") ATT_ASSERT(ka.output_size, (size_t)0, "Primary-ignorable accent key is empty") mjb_result_free(&ka); - ATT_ASSERT_STATUS(mjb_collation_key(combining_acute, 2, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_SECONDARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key(combining_acute, 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_SECONDARY, &ka, NULL), MJB_STATUS_OK, "Secondary accent key succeeds") ATT_ASSERT((int)(ka.output_size > 0), 1, "Secondary accent key retains its weight") mjb_result_free(&ka); @@ -560,11 +603,11 @@ int test_collation(void *arg) { MJB_COLLATION_SHIFTED, MJB_COLLATION_QUATERNARY) != 0), 1, "Shifted quaternary strength compares variable punctuation") - ATT_ASSERT_STATUS(mjb_collation_key("A", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_SECONDARY, &ka), + ATT_ASSERT_STATUS(mjb_collation_key("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_SECONDARY, &ka, NULL), MJB_STATUS_OK, "Secondary key for uppercase succeeds") - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_SECONDARY, &kb), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_SECONDARY, &kb, NULL), MJB_STATUS_OK, "Secondary key for lowercase succeeds") ATT_ASSERT((int)(ka.output_size == kb.output_size && memcmp(ka.output, kb.output, ka.output_size) == 0), 1, @@ -628,12 +671,12 @@ int test_collation(void *arg) { size_t output_size = sizeof(output); int order = 0; - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &result), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &result, NULL), MJB_STATUS_FEATURE_NOT_ENABLED, "Disabled collation key reports feature status") - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, output, - &output_size), + &output_size, NULL), MJB_STATUS_FEATURE_NOT_ENABLED, "Disabled collation key into reports feature status") ATT_ASSERT_STATUS(mjb_collation_compare("a", 1, MJB_ENC_UTF_8, "b", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &order), diff --git a/tests/embedded-null.c b/tests/embedded-null.c index b894d1ea..c9574cbb 100644 --- a/tests/embedded-null.c +++ b/tests/embedded-null.c @@ -17,7 +17,8 @@ static size_t count_codepoints(const char *buffer, size_t byte_length, mjb_encoding encoding) { size_t count = 0; - if(mjb_codepoint_count(buffer, byte_length, encoding, &count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(buffer, byte_length, encoding, MJB_MALFORMED_STOP, &count, NULL) != + MJB_STATUS_OK) { return SIZE_MAX; } @@ -162,16 +163,16 @@ int test_embedded_null(void *arg) { // The normalization fast path must report the resolved payload length, not SIZE_MAX. const char normalized[] = { 'A', '\0', 'B', '\0' }; mjb_result result; - ATT_ASSERT_STATUS(mjb_normalize(normalized, sizeof(normalized) - 1, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_normalize(normalized, sizeof(normalized) - 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_OK, "Normalization with explicit embedded U+0000") ATT_ASSERT(result.output_size, sizeof(normalized) - 1, "Explicit normalization preserves bytes after U+0000") ATT_ASSERT_STATUS(mjb_result_free(&result), MJB_STATUS_OK, "Free explicit normalization result") - ATT_ASSERT_STATUS(mjb_normalize(normalized, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_normalize(normalized, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_OK, "Normalization with MJB_NUL_TERMINATED") ATT_ASSERT(result.output_size, (size_t)1, "NUL-terminated normalization excludes the terminator and suffix") @@ -187,19 +188,19 @@ int test_embedded_null(void *arg) { (uint8_t)MJB_BT_NOT_SET, "Stateful segmentation requires an explicit byte length") size_t grapheme_count = 0; - ATT_ASSERT_STATUS(mjb_grapheme_count(utf8_with_nulls, 5, MJB_ENC_UTF_8, &grapheme_count), + ATT_ASSERT_STATUS(mjb_grapheme_count(utf8_with_nulls, 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &grapheme_count, NULL), MJB_STATUS_OK, "Explicit grapheme count with embedded U+0000") ATT_ASSERT(grapheme_count, (size_t)5, "UTF-8: A\\0B\\0C = 5 grapheme clusters") - ATT_ASSERT_STATUS(mjb_grapheme_count(utf8_with_nulls, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, - &grapheme_count), MJB_STATUS_OK, "NUL-terminated grapheme count") + ATT_ASSERT_STATUS(mjb_grapheme_count(utf8_with_nulls, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &grapheme_count, NULL), MJB_STATUS_OK, "NUL-terminated grapheme count") ATT_ASSERT(grapheme_count, (size_t)1, "UTF-8: NUL-terminated A\\0B\\0C = 1 grapheme cluster") size_t word_count = 0; - ATT_ASSERT_STATUS(mjb_word_count(utf8_with_nulls, 5, MJB_ENC_UTF_8, &word_count), + ATT_ASSERT_STATUS(mjb_word_count(utf8_with_nulls, 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &word_count, NULL), MJB_STATUS_OK, "Explicit word count with embedded U+0000") ATT_ASSERT(word_count, (size_t)3, "UTF-8: A\\0B\\0C = 3 words") - ATT_ASSERT_STATUS(mjb_word_count(utf8_with_nulls, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, - &word_count), MJB_STATUS_OK, "NUL-terminated word count") + ATT_ASSERT_STATUS(mjb_word_count(utf8_with_nulls, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &word_count, NULL), MJB_STATUS_OK, "NUL-terminated word count") ATT_ASSERT(word_count, (size_t)1, "UTF-8: NUL-terminated A\\0B\\0C = 1 word") return 0; diff --git a/tests/encoding.c b/tests/encoding.c index 8bd88bb6..a9433636 100644 --- a/tests/encoding.c +++ b/tests/encoding.c @@ -10,7 +10,8 @@ static size_t count_codepoints(const char *buffer, size_t byte_length, mjb_encoding encoding) { size_t count = 0; - if(mjb_codepoint_count(buffer, byte_length, encoding, &count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(buffer, byte_length, encoding, MJB_MALFORMED_STOP, &count, NULL) != + MJB_STATUS_OK) { return SIZE_MAX; } @@ -22,8 +23,8 @@ static void assert_encoding_conversion(const char *input, size_t input_size, size_t expected_size, const char *message) { mjb_result result; - ATT_ASSERT_STATUS(mjb_convert_encoding(input, input_size, input_encoding, - output_encoding, &result), + ATT_ASSERT_STATUS(mjb_convert_encoding(input, input_size, input_encoding, MJB_MALFORMED_STOP, + output_encoding, &result, NULL), MJB_STATUS_OK, message) ATT_ASSERT(result.output_size, expected_size, message) ATT_ASSERT(memcmp(result.output, expected, expected_size), 0, message) @@ -40,8 +41,8 @@ static mjb_status api_md_examples(void) { mjb_status status; // Get the length. - status = mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, NULL, - &required); + status = mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, &required, NULL); if(status != MJB_STATUS_OK) { return status; @@ -55,8 +56,8 @@ static mjb_status api_md_examples(void) { size_t capacity = required; - status = mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, - output, &capacity); + status = mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, output, &capacity, NULL); if(status != MJB_STATUS_OK) { free(output); @@ -74,7 +75,200 @@ static mjb_status api_md_examples(void) { return MJB_STATUS_OK; } +static void test_validation_and_decoding(void) { + mjb_diagnostic diagnostic = { MJB_TEXT_ERROR_OUT_OF_RANGE, 99, 99, 99 }; + + MJB_TEST_COVERAGE(mjb_string_validate); + ATT_ASSERT_STATUS(mjb_string_validate(NULL, 0, MJB_ENC_UTF_8, &diagnostic), MJB_STATUS_OK, + "NULL with zero size is well-formed") + ATT_ASSERT((unsigned int)diagnostic.error, (unsigned int)MJB_TEXT_ERROR_NONE, + "Successful validation clears the diagnostic") + ATT_ASSERT_STATUS(mjb_string_validate(NULL, 0, MJB_ENC_UNKNOWN, &diagnostic), + MJB_STATUS_INVALID_ENCODING, "Validation rejects an invalid encoding for empty input") + + const char valid_utf8[] = { 'A', '\xC3', '\xA9', '\xF0', '\x9F', '\x99', '\x82' }; + size_t offset = 0; + mjb_codepoint codepoint = MJB_CODEPOINT_NOT_VALID; + + MJB_TEST_COVERAGE(mjb_decode_next); + ATT_ASSERT_STATUS(mjb_decode_next(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode next ASCII") + ATT_ASSERT((unsigned int)codepoint, 0x41u, "Decode next ASCII codepoint") + ATT_ASSERT(offset, (size_t)1, "Decode next ASCII offset") + ATT_ASSERT_STATUS(mjb_decode_next(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode next two-byte UTF-8") + ATT_ASSERT((unsigned int)codepoint, 0xE9u, "Decode next two-byte codepoint") + ATT_ASSERT(offset, (size_t)3, "Decode next two-byte offset") + ATT_ASSERT_STATUS(mjb_decode_next(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode next four-byte UTF-8") + ATT_ASSERT((unsigned int)codepoint, 0x1F642u, "Decode next four-byte codepoint") + ATT_ASSERT(offset, sizeof(valid_utf8), "Decode next four-byte offset") + ATT_ASSERT_STATUS(mjb_decode_next(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_END_OF_INPUT, "Decode next distinguishes end of input") + + MJB_TEST_COVERAGE(mjb_decode_previous); + ATT_ASSERT_STATUS(mjb_decode_previous(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode previous four-byte UTF-8") + ATT_ASSERT((unsigned int)codepoint, 0x1F642u, "Decode previous four-byte codepoint") + ATT_ASSERT(offset, (size_t)3, "Decode previous four-byte offset") + ATT_ASSERT_STATUS(mjb_decode_previous(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode previous two-byte UTF-8") + ATT_ASSERT((unsigned int)codepoint, 0xE9u, "Decode previous two-byte codepoint") + ATT_ASSERT(offset, (size_t)1, "Decode previous two-byte offset") + ATT_ASSERT_STATUS(mjb_decode_previous(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode previous ASCII") + ATT_ASSERT((unsigned int)codepoint, 0x41u, "Decode previous ASCII codepoint") + ATT_ASSERT(offset, (size_t)0, "Decode previous ASCII offset") + ATT_ASSERT_STATUS(mjb_decode_previous(valid_utf8, sizeof(valid_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_END_OF_INPUT, "Decode previous distinguishes start of input") + + const char malformed_utf8[] = { 'A', '\x80', 'B' }; + offset = 1; + ATT_ASSERT_STATUS(mjb_decode_next(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Stop policy reports malformed UTF-8") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + "Stop policy reports unexpected continuation") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Stop diagnostic byte offset") + ATT_ASSERT(diagnostic.byte_length, (size_t)1, "Stop diagnostic byte length") + ATT_ASSERT(diagnostic.code_unit_offset, (size_t)1, "Stop diagnostic code-unit offset") + ATT_ASSERT(offset, (size_t)2, "Stop policy advances over malformed subsequence") + + offset = 1; + ATT_ASSERT_STATUS(mjb_decode_next(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Replace policy decodes malformed UTF-8") + ATT_ASSERT((unsigned int)codepoint, (unsigned int)MJB_CODEPOINT_REPLACEMENT, + "Replace policy emits U+FFFD") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + "Replace policy retains the diagnostic") + + offset = 1; + ATT_ASSERT_STATUS(mjb_decode_next(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Skip policy resumes after malformed UTF-8") + ATT_ASSERT((unsigned int)codepoint, (unsigned int)'B', "Skip policy returns the next codepoint") + ATT_ASSERT(offset, sizeof(malformed_utf8), "Skip policy advances through the next codepoint") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + "Skip policy retains the diagnostic") + + offset = sizeof(malformed_utf8); + ATT_ASSERT_STATUS(mjb_decode_previous(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode previous after malformed UTF-8") + ATT_ASSERT((unsigned int)codepoint, (unsigned int)'B', "Decode previous trailing codepoint") + ATT_ASSERT_STATUS(mjb_decode_previous(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode previous replaces malformed UTF-8") + ATT_ASSERT((unsigned int)codepoint, (unsigned int)MJB_CODEPOINT_REPLACEMENT, + "Decode previous replacement codepoint") + ATT_ASSERT(offset, (size_t)1, "Decode previous malformed offset") + + const char truncated_utf8[] = { '\xE2', '\x82' }; + offset = sizeof(truncated_utf8); + ATT_ASSERT_STATUS(mjb_decode_previous(truncated_utf8, sizeof(truncated_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Decode previous replaces a complete truncated subsequence") + ATT_ASSERT((unsigned int)codepoint, (unsigned int)MJB_CODEPOINT_REPLACEMENT, + "Decode previous truncated replacement codepoint") + ATT_ASSERT(offset, (size_t)0, "Decode previous consumes the complete truncated subsequence") + ATT_ASSERT(diagnostic.byte_length, sizeof(truncated_utf8), + "Decode previous reports the complete truncated subsequence") + + struct malformed_case { + const char *input; + size_t size; + mjb_encoding encoding; + mjb_text_error error; + size_t error_size; + } malformed_cases[] = { + { "\x80", 1, MJB_ENC_ASCII, MJB_TEXT_ERROR_NON_ASCII, 1 }, + { "\xFF", 1, MJB_ENC_UTF_8, MJB_TEXT_ERROR_INVALID_LEADING_BYTE, 1 }, + { "\xE2(A", 3, MJB_ENC_UTF_8, MJB_TEXT_ERROR_MISSING_CONTINUATION, 1 }, + { "\xE0\x80\x80", 3, MJB_ENC_UTF_8, MJB_TEXT_ERROR_OVERLONG_SEQUENCE, 1 }, + { "\xED\xA0\x80", 3, MJB_ENC_UTF_8, MJB_TEXT_ERROR_SURROGATE, 1 }, + { "\xF4\x90\x80\x80", 4, MJB_ENC_UTF_8, MJB_TEXT_ERROR_OUT_OF_RANGE, 1 }, + { "\xE2\x82", 2, MJB_ENC_UTF_8, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, 2 }, + { "A", 1, MJB_ENC_UTF_16LE, MJB_TEXT_ERROR_TRUNCATED_CODE_UNIT, 1 }, + { "\x00\xD8", 2, MJB_ENC_UTF_16LE, MJB_TEXT_ERROR_UNPAIRED_SURROGATE, 2 }, + { "\x00\xD8\x00\x00", 4, MJB_ENC_UTF_32LE, MJB_TEXT_ERROR_SURROGATE, 4 }, + { "\x00\x00\x11\x00", 4, MJB_ENC_UTF_32LE, MJB_TEXT_ERROR_OUT_OF_RANGE, 4 }, + }; + + for(size_t i = 0; i < sizeof(malformed_cases) / sizeof(malformed_cases[0]); ++i) { + diagnostic.error = MJB_TEXT_ERROR_NONE; + ATT_ASSERT_STATUS(mjb_string_validate(malformed_cases[i].input, malformed_cases[i].size, + malformed_cases[i].encoding, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Validation rejects malformed encoding case") + ATT_ASSERT((unsigned int)diagnostic.error, (unsigned int)malformed_cases[i].error, + "Validation reports precise malformed encoding kind") + ATT_ASSERT(diagnostic.byte_length, malformed_cases[i].error_size, + "Validation reports malformed subsequence size") + } + + const char generic_utf16[] = { '\xFF', '\xFE', 'A', '\0' }; + offset = 0; + ATT_ASSERT_STATUS(mjb_decode_next(generic_utf16, sizeof(generic_utf16), MJB_ENC_UTF_16, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_OK, "Generic UTF-16 decoder consumes BOM") + ATT_ASSERT((unsigned int)codepoint, (unsigned int)'A', "Generic UTF-16 decoded codepoint") + ATT_ASSERT(offset, sizeof(generic_utf16), "Generic UTF-16 decoded offset") + offset = 1; + ATT_ASSERT_STATUS(mjb_decode_next(generic_utf16, sizeof(generic_utf16), MJB_ENC_UTF_16, + MJB_MALFORMED_STOP, &offset, &codepoint, &diagnostic), + MJB_STATUS_INVALID_ARGUMENT, "Generic UTF-16 rejects an offset inside its BOM") + + size_t count = 99; + ATT_ASSERT_STATUS(mjb_codepoint_count(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, &count, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Codepoint count stop policy") + ATT_ASSERT(count, (size_t)0, "Codepoint count is reset after malformed input") + ATT_ASSERT_STATUS(mjb_codepoint_count(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &count, &diagnostic), + MJB_STATUS_OK, "Codepoint count replace policy") + ATT_ASSERT(count, (size_t)3, "Replacement counts as a codepoint") + ATT_ASSERT_STATUS(mjb_codepoint_count(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, &count, &diagnostic), + MJB_STATUS_OK, "Codepoint count skip policy") + ATT_ASSERT(count, (size_t)2, "Skipped malformed input is not counted") + + mjb_result converted; + ATT_ASSERT_STATUS(mjb_convert_encoding(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &converted, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Same-encoding conversion validates in stop mode") + ATT_ASSERT_STATUS(mjb_convert_encoding(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, MJB_ENC_UTF_8, &converted, &diagnostic), + MJB_STATUS_OK, "Same-encoding conversion replaces malformed input") + ATT_ASSERT(converted.output_size, (size_t)5, "Replacement conversion output size") + ATT_ASSERT(memcmp(converted.output, "A\xEF\xBF\xBD" + "B", + converted.output_size), + 0, "Replacement conversion output") + (void)mjb_result_free(&converted); + + ATT_ASSERT_STATUS(mjb_convert_encoding(malformed_utf8, sizeof(malformed_utf8), MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, MJB_ENC_UTF_8, &converted, &diagnostic), + MJB_STATUS_OK, "Same-encoding conversion skips malformed input") + ATT_ASSERT(converted.output_size, (size_t)2, "Skip conversion output size") + ATT_ASSERT(memcmp(converted.output, "AB", converted.output_size), 0, + "Skip conversion output") + (void)mjb_result_free(&converted); +} + int test_encoding(void *arg) { + test_validation_and_decoding(); + ATT_ASSERT((unsigned int)mjb_detect_encoding(0, 10), (unsigned int)MJB_ENC_UNKNOWN, "Void unknown string") ATT_ASSERT((unsigned int)mjb_detect_encoding("", 0), (unsigned int)MJB_ENC_UNKNOWN, @@ -109,9 +303,8 @@ int test_encoding(void *arg) { (unsigned int)(MJB_ENC_UTF_32 | MJB_ENC_UTF_32LE | MJB_ENC_UTF_16 | MJB_ENC_UTF_16LE), "UTF-32-LE BOM") - ATT_ASSERT(mjb_is_ascii("", 0), false, "Void ASCII string") - ATT_ASSERT(mjb_is_ascii("", 0), false, "Void ASCII length") - ATT_ASSERT(mjb_is_ascii(0, 0), false, "Void ASCII string and length") + ATT_ASSERT(mjb_is_ascii("", 0), true, "Empty ASCII string") + ATT_ASSERT(mjb_is_ascii(0, 0), true, "NULL with zero size is empty ASCII") const char *test10 = "The quick brown fox jumps over the lazy dog"; ATT_ASSERT(mjb_is_ascii(test10, 43), true, "Valid string and length") @@ -140,9 +333,9 @@ int test_encoding(void *arg) { const char *utf8_test = ""; - ATT_ASSERT(mjb_is_utf8(NULL, 0), false, "Void UTF-8 string") - ATT_ASSERT(mjb_is_utf8("", 0), false, "Empty UTF-8 \"\" string") - ATT_ASSERT(mjb_is_utf8(utf8_test, strlen(utf8_test)), false, "Empty UTF-8 string") + ATT_ASSERT(mjb_is_utf8(NULL, 0), true, "NULL with zero size is empty UTF-8") + ATT_ASSERT(mjb_is_utf8("", 0), true, "Empty UTF-8 \"\" string") + ATT_ASSERT(mjb_is_utf8(utf8_test, strlen(utf8_test)), true, "Empty UTF-8 string") utf8_test = "Hello, world!"; ATT_ASSERT(mjb_is_utf8(utf8_test, strlen(utf8_test)), true, "Simple ASCII") @@ -203,8 +396,8 @@ int test_encoding(void *arg) { "Various Unicode punctuation and symbols") // UTF-16 tests - ATT_ASSERT(mjb_is_utf16(NULL, 0), false, "Void UTF-16 string") - ATT_ASSERT(mjb_is_utf16("", 0), false, "Empty UTF-16 string") + ATT_ASSERT(mjb_is_utf16(NULL, 0), true, "NULL with zero size is empty UTF-16") + ATT_ASSERT(mjb_is_utf16("", 0), true, "Empty UTF-16 string") ATT_ASSERT(mjb_is_utf16("A", 1), false, "Odd-length UTF-16 string") ATT_ASSERT(mjb_is_utf16("", 1), false, "Odd-length empty UTF-16 string") @@ -409,22 +602,23 @@ int test_encoding(void *arg) { char test_description[64]; mjb_result boundary_result; - ATT_ASSERT_STATUS(mjb_convert_encoding(NULL, 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, - &boundary_result), + ATT_ASSERT_STATUS(mjb_convert_encoding(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &boundary_result, NULL), MJB_STATUS_INVALID_ARGUMENT, "Convert encoding rejects NULL buffer") - ATT_ASSERT_STATUS(mjb_convert_encoding("", 0, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, NULL), + ATT_ASSERT_STATUS(mjb_convert_encoding("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Convert encoding rejects NULL result") MJB_TEST_COVERAGE(mjb_convert_encoding_into); size_t into_size = 123; - ATT_ASSERT_STATUS(mjb_convert_encoding_into(NULL, 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, NULL, - &into_size), + ATT_ASSERT_STATUS(mjb_convert_encoding_into(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Convert encoding into rejects NULL buffer") ATT_ASSERT(into_size, (size_t)0, "Convert encoding into resets size for invalid input") - ATT_ASSERT_STATUS(mjb_convert_encoding_into("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, NULL, - NULL), + ATT_ASSERT_STATUS(mjb_convert_encoding_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Convert encoding into rejects NULL output size") const char into_input[] = "caf\xC3\xA9"; @@ -432,7 +626,7 @@ int test_encoding(void *arg) { into_size = 0; ATT_ASSERT_STATUS(mjb_convert_encoding_into(into_input, strlen(into_input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, NULL, &into_size), + MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, NULL, &into_size, NULL), MJB_STATUS_OK, "Query encoding conversion output size") ATT_ASSERT(into_size, sizeof(into_expected), "Encoding conversion required payload size") @@ -441,7 +635,7 @@ int test_encoding(void *arg) { into_size = sizeof(into_expected) - 1; ATT_ASSERT_STATUS(mjb_convert_encoding_into(into_input, strlen(into_input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, into_output, &into_size), + MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, into_output, &into_size, NULL), MJB_STATUS_OUTPUT_TOO_SMALL, "Convert encoding reports a small output buffer") ATT_ASSERT(into_size, sizeof(into_expected), "Small output buffer reports required size") @@ -452,7 +646,7 @@ int test_encoding(void *arg) { into_size = sizeof(into_expected); ATT_ASSERT_STATUS(mjb_convert_encoding_into(into_input, strlen(into_input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, into_output, &into_size), + MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, into_output, &into_size, NULL), MJB_STATUS_OK, "Convert encoding into exact-size output buffer") ATT_ASSERT(into_size, sizeof(into_expected), "Convert encoding into written payload size") ATT_ASSERT(memcmp(into_output, into_expected, sizeof(into_expected)), 0, @@ -462,8 +656,8 @@ int test_encoding(void *arg) { unsigned char same_encoding_output[] = { 0, 0xA5 }; into_size = 1; - ATT_ASSERT_STATUS(mjb_convert_encoding_into("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_8, - same_encoding_output, &into_size), + ATT_ASSERT_STATUS(mjb_convert_encoding_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, same_encoding_output, &into_size, NULL), MJB_STATUS_OK, "Convert encoding into copies unchanged encoding") ATT_ASSERT((unsigned int)same_encoding_output[0], (unsigned int)'A', "Convert encoding into unchanged byte") @@ -474,22 +668,24 @@ int test_encoding(void *arg) { mjb_result ascii_result; ATT_ASSERT_STATUS(mjb_convert_encoding(utf16le_ascii, sizeof(utf16le_ascii), - MJB_ENC_UTF_16LE, MJB_ENC_ASCII, &ascii_result), + MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, MJB_ENC_ASCII, &ascii_result, + NULL), MJB_STATUS_OK, "Convert UTF-16LE ASCII text to ASCII") ATT_ASSERT(ascii_result.transformed, true, "Convert UTF-16LE ASCII text is transformed") ATT_ASSERT(ascii_result.output_size, (size_t)2, "Convert UTF-16LE ASCII text size") ATT_ASSERT(ascii_result.output, "en", "Convert UTF-16LE ASCII text output") (void)mjb_result_free(&ascii_result); - ATT_ASSERT_STATUS(mjb_convert_encoding("\xC3\xA9", 2, MJB_ENC_UTF_8, MJB_ENC_ASCII, - &ascii_result), + ATT_ASSERT_STATUS(mjb_convert_encoding("\xC3\xA9", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_ASCII, &ascii_result, NULL), MJB_STATUS_UNSUPPORTED, "Convert UTF-8 non-ASCII text to ASCII") const char utf16le_smile[] = { '\x3D', '\xD8', '\x42', '\xDE' }; const char utf16be_smile[] = { '\xD8', '\x3D', '\xDE', '\x42' }; ATT_ASSERT_STATUS(mjb_convert_encoding(utf16le_smile, sizeof(utf16le_smile), - MJB_ENC_UTF_16LE, MJB_ENC_UTF_8, &ascii_result), + MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &ascii_result, + NULL), MJB_STATUS_OK, "Convert UTF-16LE surrogate pair to UTF-8") ATT_ASSERT(ascii_result.output_size, (size_t)4, "Convert UTF-16LE surrogate pair to UTF-8 size") ATT_ASSERT(memcmp(ascii_result.output, "\xF0\x9F\x99\x82", ascii_result.output_size), 0, @@ -497,7 +693,8 @@ int test_encoding(void *arg) { (void)mjb_result_free(&ascii_result); ATT_ASSERT_STATUS(mjb_convert_encoding(utf16be_smile, sizeof(utf16be_smile), - MJB_ENC_UTF_16BE, MJB_ENC_UTF_8, &ascii_result), + MJB_ENC_UTF_16BE, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &ascii_result, + NULL), MJB_STATUS_OK, "Convert UTF-16BE surrogate pair to UTF-8") ATT_ASSERT(ascii_result.output_size, (size_t)4, "Convert UTF-16BE surrogate pair to UTF-8 size") ATT_ASSERT(memcmp(ascii_result.output, "\xF0\x9F\x99\x82", ascii_result.output_size), 0, @@ -590,18 +787,18 @@ int test_encoding(void *arg) { size_t rejected_count = 6251; ATT_ASSERT_STATUS(mjb_codepoint_count(utf32le_bom_a, sizeof(utf32le_bom_a), detected_utf32le, - &rejected_count), MJB_STATUS_INVALID_ENCODING, + MJB_MALFORMED_STOP, &rejected_count, NULL), MJB_STATUS_INVALID_ENCODING, "Codepoint count rejects a multi-encoding detection mask") ATT_ASSERT(rejected_count, (size_t)0, "Codepoint count is zero after a detection mask") ATT_ASSERT(count_codepoints(utf32le_bom_a, sizeof(utf32le_bom_a), MJB_ENC_UTF_32LE), (size_t)2, "Length explicit UTF-32LE preserves U+FEFF") ATT_ASSERT_STATUS(mjb_convert_encoding(utf16be_plain_a, sizeof(utf16be_plain_a), - MJB_ENC_UTF_16, MJB_ENC_UTF_8, &ascii_result), + MJB_ENC_UTF_16, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &ascii_result, NULL), MJB_STATUS_INVALID_ENCODING, "Convert generic UTF-16 without BOM rejects unknown byte order") - ATT_ASSERT_STATUS(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16, - &ascii_result), + ATT_ASSERT_STATUS(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16, &ascii_result, NULL), MJB_STATUS_INVALID_ENCODING, "Convert generic UTF-16 output rejects unknown byte order") MJB_TEST_COVERAGE(mjb_convert_encoding); @@ -609,7 +806,7 @@ int test_encoding(void *arg) { for(size_t to = 0; to < 5; ++to) { mjb_result convert_result; mjb_status status = mjb_convert_encoding(hello_strings[from], hello_strings_sizes[from], - encodings[from], encodings[to], &convert_result); + encodings[from], MJB_MALFORMED_STOP, encodings[to], &convert_result, NULL); snprintf(test_description, 64, "%s to %s", output_encodings[from], output_encodings[to]); diff --git a/tests/example.c b/tests/example.c index b1a310eb..eac95c3b 100644 --- a/tests/example.c +++ b/tests/example.c @@ -33,8 +33,8 @@ int test_example(void *arg) { const char *input = "Cafe\xCC\x81"; // "Cafe" + U+0301 COMBINING ACUTE ACCENT mjb_result result; - if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { + if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_normalize test failed") // Added by the script return 1; } @@ -53,8 +53,8 @@ int test_example(void *arg) { const char *input = "Cafe\xCC\x81"; // "Cafe" + U+0301 COMBINING ACUTE ACCENT size_t output_size = 0; - if(mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, NULL, &output_size) != MJB_STATUS_OK) { + if(mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_normalize_into test failed") // Added by the script return 1; } @@ -62,7 +62,8 @@ int test_example(void *arg) { char output[5]; if(output_size > sizeof(output) || mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, output, &output_size, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_normalize_into test failed") // Added by the script return 1; } @@ -80,7 +81,8 @@ int test_example(void *arg) { mjb_result result; if(mjb_filter(mixed_whitespace, strlen(mixed_whitespace), MJB_ENC_UTF_8, - MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, &result) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, &result, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_filter test failed") // Added by the script return 1; } @@ -94,8 +96,8 @@ int test_example(void *arg) { const char *controls = "\x1\x2\t\n\v\f\r\x1f"; - if(mjb_filter(controls, strlen(controls), MJB_ENC_UTF_8, MJB_FILTER_CONTROLS, - MJB_ENC_UTF_8, &result) != MJB_STATUS_OK) { + if(mjb_filter(controls, strlen(controls), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_CONTROLS, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_filter test failed") // Added by the script return 1; } @@ -114,8 +116,9 @@ int test_example(void *arg) { const char *input = "Hello\t\t\nworld"; size_t output_size = 0; - if(mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, MJB_FILTER_COLLAPSE_SPACES, - MJB_ENC_UTF_8, NULL, &output_size) != MJB_STATUS_OK) { + if(mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, NULL, &output_size, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_filter_into test failed") // Added by the script return 1; } @@ -123,7 +126,8 @@ int test_example(void *arg) { char output[11]; if(output_size > sizeof(output) || mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, output, + &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_filter_into test failed") // Added by the script return 1; } @@ -140,8 +144,8 @@ int test_example(void *arg) { const char *input = "Stra\xC3\x9F" "e\xC2\xAD"; mjb_result result; - if(mjb_nfkc_casefold(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { + if(mjb_nfkc_casefold(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_nfkc_casefold test failed") // Added by the script return 1; } @@ -159,8 +163,8 @@ int test_example(void *arg) { const char *input = "Stra\xC3\x9F" "e\xC2\xAD"; size_t output_size = 0; - if(mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, - NULL, &output_size) != MJB_STATUS_OK) { + if(mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_nfkc_casefold_into test failed") // Added by the script return 1; } @@ -168,7 +172,7 @@ int test_example(void *arg) { char output[7]; if(output_size > sizeof(output) || mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, output, &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_nfkc_casefold_into test failed") // Added by the script return 1; } @@ -287,6 +291,19 @@ int test_example(void *arg) { ATT_ASSERT(test_buffer, "UTF-16: yes", "mjb_is_utf16 test failed") // Added by the script } +{ + // Example for mjb_string_validate + MJB_TEST_COVERAGE(mjb_string_validate); // Added by the script + const char invalid[] = "\xE2\x82"; + mjb_diagnostic diagnostic; + + if(mjb_string_validate(invalid, sizeof(invalid) - 1, MJB_ENC_UTF_8, + &diagnostic) != MJB_STATUS_MALFORMED_INPUT || diagnostic.byte_offset != 0) { + ATT_ASSERT(0, 1, "mjb_string_validate test failed") // Added by the script + return 1; + } +} + { // Example for mjb_codepoint_count MJB_TEST_COVERAGE(mjb_codepoint_count); // Added by the script @@ -296,7 +313,8 @@ int test_example(void *arg) { const char utf16le[] = "H\0\xE9\0l\0l\0\xF6\0"; // 10 bytes size_t count; - if(mjb_codepoint_count(utf8, 7, MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(utf8, 7, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_codepoint_count test failed") // Added by the script return 1; } @@ -306,7 +324,8 @@ int test_example(void *arg) { snprintf(test_buffer, sizeof(test_buffer), "%zu UTF-8 characters", count); // Added by the script ATT_ASSERT(test_buffer, "5 UTF-8 characters", "mjb_codepoint_count test failed") // Added by the script - if(mjb_codepoint_count(utf16le, 10, MJB_ENC_UTF_16LE, &count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(utf16le, 10, MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_codepoint_count test failed") // Added by the script return 1; } @@ -320,7 +339,8 @@ int test_example(void *arg) { { // Example for mjb_for_each_codepoint MJB_TEST_COVERAGE(mjb_for_each_codepoint); // Added by the script - mjb_status status = mjb_for_each_codepoint("ABC", 3, MJB_ENC_UTF_8, NULL); + mjb_status status = mjb_for_each_codepoint("ABC", 3, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, NULL, NULL); // A callback is required: yes bool callback_required = status == MJB_STATUS_INVALID_ARGUMENT; @@ -459,8 +479,8 @@ int test_example(void *arg) { const char *input = "caf\xC3\xA9"; mjb_result result; - if(mjb_convert_encoding(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, &result) != MJB_STATUS_OK) { + if(mjb_convert_encoding(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_convert_encoding test failed") // Added by the script return 1; } @@ -478,8 +498,8 @@ int test_example(void *arg) { const char *input = "caf\xC3\xA9"; size_t output_size = 0; - if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, NULL, &output_size) != MJB_STATUS_OK) { + if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_convert_encoding_into test failed") // Added by the script return 1; } @@ -487,7 +507,8 @@ int test_example(void *arg) { unsigned char output[8]; if(output_size > sizeof(output) || mjb_convert_encoding_into(input, strlen(input), - MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, output, &output_size) != MJB_STATUS_OK) { + MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, output, &output_size, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_convert_encoding_into test failed") // Added by the script return 1; } @@ -544,7 +565,8 @@ int test_example(void *arg) { mjb_result key; if(mjb_collation_key("r\xC3\xA9sum\xC3\xA9", 8, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &key) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &key, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_collation_key test failed") // Added by the script return 1; } @@ -564,8 +586,9 @@ int test_example(void *arg) { const char *input = "r\xC3\xA9sum\xC3\xA9"; size_t output_size = 0; - if(mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, NULL, &output_size) != MJB_STATUS_OK) { + if(mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &output_size, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_collation_key_into test failed") // Added by the script return 1; } @@ -573,8 +596,9 @@ int test_example(void *arg) { unsigned char output[64]; if(output_size > sizeof(output) || mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, output, - &output_size) != MJB_STATUS_OK) { + &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_collation_key_into test failed") // Added by the script return 1; } @@ -592,8 +616,8 @@ int test_example(void *arg) { const char *input = "Stra\xC3\x9F""e"; // "Straße" mjb_result result; - if(mjb_map_case(input, strlen(input), MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { + if(mjb_map_case(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_CASE_UPPER, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_map_case test failed") // Added by the script return 1; } @@ -612,8 +636,8 @@ int test_example(void *arg) { const char *input = "Stra\xC3\x9F""e"; // "Straße" size_t output_size = 0; - if(mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - NULL, &output_size) != MJB_STATUS_OK) { + if(mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_CASE_UPPER, MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_map_case_into test failed") // Added by the script return 1; } @@ -621,7 +645,8 @@ int test_example(void *arg) { char output[7]; if(output_size > sizeof(output) || mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_CASE_UPPER, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_8, output, &output_size, + NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_map_case_into test failed") // Added by the script return 1; } @@ -800,7 +825,8 @@ int test_example(void *arg) { const char *input = "Hello. How are you? Fine!"; size_t count; - if(mjb_sentence_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { + if(mjb_sentence_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_sentence_count test failed") // Added by the script return 1; } @@ -848,7 +874,8 @@ int test_example(void *arg) { const char *input = "A\xF0\x9F\x87\xAE\xF0\x9F\x87\xB9"; // A🇮🇹 size_t count; - if(mjb_grapheme_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { + if(mjb_grapheme_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_grapheme_count test failed") // Added by the script return 1; } @@ -890,7 +917,8 @@ int test_example(void *arg) { const char *input = "Hello, world! It works."; size_t count; - if(mjb_word_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { + if(mjb_word_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_word_count test failed") // Added by the script return 1; } @@ -1417,8 +1445,8 @@ int test_example(void *arg) { const char *input = "A\xE7\x95\x8C"; // A界 size_t width; - if(mjb_terminal_width(input, strlen(input), MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &width) != MJB_STATUS_OK) { + if(mjb_terminal_width(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &width, NULL) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_terminal_width test failed") // Added by the script return 1; } @@ -1496,8 +1524,9 @@ int test_example(void *arg) { MJB_TEST_COVERAGE(mjb_result_free); // Added by the script mjb_result result; - if(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, - &result) != MJB_STATUS_OK || mjb_result_free(&result) != MJB_STATUS_OK) { + if(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK || + mjb_result_free(&result) != MJB_STATUS_OK) { ATT_ASSERT(0, 1, "mjb_result_free test failed") // Added by the script return 1; } diff --git a/tests/ext/cpp/normalization.cpp b/tests/ext/cpp/normalization.cpp index 1a2c1f1b..4d1f8840 100644 --- a/tests/ext/cpp/normalization.cpp +++ b/tests/ext/cpp/normalization.cpp @@ -47,10 +47,65 @@ int test_cpp_normalization(void *arg) { const std::string utf16le("a\0b\0", 4); ATT_ASSERT(mjb::is_utf16(utf16le), true, "is_utf16") + mjb_diagnostic diagnostic; + ATT_ASSERT_STATUS(mjb::validate_string("A\x80", MJB_ENC_UTF_8, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "validate_string malformed input") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + "validate_string diagnostic") + + const std::string decodable("A\x80" + "B", + 3); + size_t offset = 0; + mjb_codepoint codepoint = MJB_CODEPOINT_NOT_VALID; + ATT_ASSERT_STATUS(mjb::decode_next(decodable, offset, codepoint), MJB_STATUS_OK, + "decode_next valid input") + ATT_ASSERT(codepoint, (mjb_codepoint)'A', "decode_next codepoint") + ATT_ASSERT_STATUS(mjb::decode_next(decodable, offset, codepoint, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &diagnostic), + MJB_STATUS_OK, "decode_next replacement policy") + ATT_ASSERT(codepoint, (mjb_codepoint)MJB_CODEPOINT_REPLACEMENT, + "decode_next replacement codepoint") + offset = decodable.size(); + ATT_ASSERT_STATUS(mjb::decode_previous(decodable, offset, codepoint), MJB_STATUS_OK, + "decode_previous valid input") + ATT_ASSERT(codepoint, (mjb_codepoint)'B', "decode_previous codepoint") + + ATT_ASSERT(mjb::nfc(decodable, MJB_ENC_UTF_8, MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE, + &diagnostic), + std::string("A\xEF\xBF\xBD" + "B"), + "nfc replacement policy") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "nfc replacement diagnostic") + ATT_ASSERT(mjb::nfc(decodable, MJB_ENC_UTF_8, MJB_ENC_UTF_8, MJB_MALFORMED_SKIP), + std::string("AB"), "nfc skip policy") + ATT_ASSERT(mjb::nfkc_casefold(decodable, MJB_ENC_UTF_8, MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP), + std::string("ab"), "nfkc_casefold skip policy") + ATT_ASSERT(mjb::uppercase(decodable, MJB_ENC_UTF_8, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE), + std::string("A\xEF\xBF\xBD" + "B"), + "uppercase replacement policy") + ATT_ASSERT(mjb::grapheme_count(decodable, MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE), + (size_t)3, "grapheme_count replacement policy") + ATT_ASSERT(mjb::sentence_count(decodable, MJB_ENC_UTF_8, MJB_MALFORMED_SKIP), (size_t)1, + "sentence_count skip policy") + ATT_ASSERT(mjb::word_count(decodable, MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE), (size_t)2, + "word_count replacement policy") + ATT_ASSERT(mjb::terminal_width(decodable, MJB_TERMINAL_WIDTH_NARROW, MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP), + (size_t)2, "terminal_width skip policy") + ATT_ASSERT(mjb::codepoint_count("caf\xC3\xA9"), (size_t)4, "codepoint_count") ATT_ASSERT(mjb::convert_encoding("\xC3\xA9", MJB_ENC_UTF_8, MJB_ENC_UTF_16LE), std::string("\xE9\0", 2), "convert_encoding") #if MJB_FEATURE_COLLATION + ATT_ASSERT(mjb::collation_key(decodable, + mjb::CollationVariableWeighting::NonIgnorable, + mjb::CollationStrength::Tertiary, MJB_ENC_UTF_8, MJB_MALFORMED_SKIP), + mjb::collation_key("AB"), "collation_key skip policy") ATT_ASSERT(mjb::compare("a", "b") < 0, true, "compare") ATT_ASSERT(mjb::compare("A", "a", mjb::CollationVariableWeighting::NonIgnorable, mjb::CollationStrength::Secondary), diff --git a/tests/filter.c b/tests/filter.c index 8a40f015..8a0e533d 100644 --- a/tests/filter.c +++ b/tests/filter.c @@ -10,20 +10,21 @@ static void assert_filter_into_matches(const char *input, size_t input_size, mjb mjb_filter_flags filters, mjb_encoding output_encoding, const char *message) { mjb_result allocated; - ATT_ASSERT_STATUS(mjb_filter(input, input_size, encoding, filters, output_encoding, &allocated), + ATT_ASSERT_STATUS(mjb_filter(input, input_size, encoding, MJB_MALFORMED_REPLACE, filters, + output_encoding, &allocated, NULL), MJB_STATUS_OK, message) size_t required = 0; - ATT_ASSERT_STATUS(mjb_filter_into(input, input_size, encoding, filters, output_encoding, NULL, - &required), + ATT_ASSERT_STATUS(mjb_filter_into(input, input_size, encoding, MJB_MALFORMED_REPLACE, filters, + output_encoding, NULL, &required, NULL), MJB_STATUS_OK, message) ATT_ASSERT(required, allocated.output_size, message) unsigned char output[256]; size_t capacity = sizeof(output); ATT_ASSERT(required <= capacity, true, message) - ATT_ASSERT_STATUS(mjb_filter_into(input, input_size, encoding, filters, output_encoding, output, - &capacity), + ATT_ASSERT_STATUS(mjb_filter_into(input, input_size, encoding, MJB_MALFORMED_REPLACE, filters, + output_encoding, output, &capacity, NULL), MJB_STATUS_OK, message) ATT_ASSERT(capacity, allocated.output_size, message) ATT_ASSERT(memcmp(output, allocated.output, capacity), 0, message) @@ -39,18 +40,23 @@ int test_filter(void *arg) { size_t into_size = 9; - ATT_ASSERT_STATUS(mjb_filter_into(NULL, 1, enc, MJB_FILTER_NONE, enc, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_filter_into(NULL, 1, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, NULL, &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Filter into rejects NULL buffer") ATT_ASSERT(into_size, (size_t)0, "Filter into resets size for invalid input") - ATT_ASSERT_STATUS(mjb_filter_into("A", 1, enc, MJB_FILTER_NONE, enc, NULL, NULL), + ATT_ASSERT_STATUS(mjb_filter_into("A", 1, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, NULL, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Filter into rejects NULL output size") + into_size = 9; + ATT_ASSERT_STATUS(mjb_filter_into("A", 1, enc, (mjb_malformed_policy)99, + MJB_FILTER_NONE, enc, NULL, &into_size, NULL), + MJB_STATUS_INVALID_ARGUMENT, "Filter into rejects invalid malformed policy") + ATT_ASSERT(into_size, (size_t)0, "Invalid filter policy resets output size") const char *into_input = "Hello\t\t\nworld"; const char *into_expected = "Hello world"; into_size = 0; - ATT_ASSERT_STATUS(mjb_filter_into(into_input, strlen(into_input), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_filter_into(into_input, strlen(into_input), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, NULL, &into_size, NULL), MJB_STATUS_OK, "Query filtered output size") ATT_ASSERT(into_size, strlen(into_expected), "Filtered output required payload size") @@ -60,16 +66,16 @@ int test_filter(void *arg) { memset(untouched_output, 0xA5, sizeof(untouched_output)); into_size = strlen(into_expected) - 1; - ATT_ASSERT_STATUS(mjb_filter_into(into_input, strlen(into_input), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, into_output, &into_size), + ATT_ASSERT_STATUS(mjb_filter_into(into_input, strlen(into_input), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, into_output, &into_size, NULL), MJB_STATUS_OUTPUT_TOO_SMALL, "Filter into reports a small output buffer") ATT_ASSERT(into_size, strlen(into_expected), "Small filter output reports required size") ATT_ASSERT(memcmp(into_output, untouched_output, sizeof(into_output)), 0, "Small filter output buffer is not modified") into_size = strlen(into_expected); - ATT_ASSERT_STATUS(mjb_filter_into(into_input, strlen(into_input), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, into_output, &into_size), + ATT_ASSERT_STATUS(mjb_filter_into(into_input, strlen(into_input), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, into_output, &into_size, NULL), MJB_STATUS_OK, "Filter into exact-size output buffer") ATT_ASSERT(into_size, strlen(into_expected), "Filter into written payload size") ATT_ASSERT(memcmp(into_output, into_expected, into_size), 0, "Filter into output bytes") @@ -80,9 +86,9 @@ int test_filter(void *arg) { const char *normalize_expected = "Caf\xC3\xA9"; into_size = sizeof(into_output); - ATT_ASSERT_STATUS(mjb_filter_into(normalize_input, strlen(normalize_input), enc, + ATT_ASSERT_STATUS(mjb_filter_into(normalize_input, strlen(normalize_input), enc, MJB_MALFORMED_REPLACE, (mjb_filter_flags)(MJB_FILTER_NORMALIZE | MJB_FILTER_SPACES), enc, - into_output, &into_size), + into_output, &into_size, NULL), MJB_STATUS_OK, "Filter into normalizes before filtering") ATT_ASSERT(into_size, strlen(normalize_expected), "Normalized filter into size") ATT_ASSERT(memcmp(into_output, normalize_expected, into_size), 0, @@ -103,39 +109,39 @@ int test_filter(void *arg) { assert_filter_into_matches(normalize_input, strlen(normalize_input), enc, MJB_FILTER_NORMALIZE, enc, "Filter into normalization matches allocating filter"); - ATT_ASSERT_STATUS(mjb_filter(NULL, 1, enc, MJB_FILTER_NONE, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(NULL, 1, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_INVALID_ARGUMENT, "Filter rejects NULL buffer") - ATT_ASSERT_STATUS(mjb_filter("", 0, enc, MJB_FILTER_NONE, enc, NULL), + ATT_ASSERT_STATUS(mjb_filter("", 0, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Filter rejects NULL result") - ATT_ASSERT_STATUS(mjb_filter("", 0, enc, MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_filter("", 0, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "Filter empty string") ATT_ASSERT(result.output, (char *)"", "Filter empty string output") ATT_ASSERT(result.output_size, 0, "Filter empty string size") ATT_ASSERT(result.transformed, false, "Filter empty string transformed") mjb_result_free(&result); - ATT_ASSERT_STATUS(mjb_filter("", 0, enc, MJB_FILTER_NORMALIZE, enc, &result), + ATT_ASSERT_STATUS(mjb_filter("", 0, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NORMALIZE, enc, &result, NULL), MJB_STATUS_OK, "Filter normalize empty string") ATT_ASSERT(result.output, (char *)"", "Filter normalize empty string output") ATT_ASSERT(result.output_size, 0, "Filter normalize empty string size") ATT_ASSERT(result.transformed, false, "Filter normalize empty string transformed") mjb_result_free(&result); - ATT_ASSERT_STATUS(mjb_filter(" ", 0, enc, MJB_FILTER_SPACES, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(" ", 0, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Filter spaces") ATT_ASSERT(result.output, (char *)" ", "Filter spaces output") mjb_result_free(&result); - ATT_ASSERT_STATUS(mjb_filter("A", 1, enc, MJB_FILTER_NONE, MJB_ENC_UTF_16LE, &result), + ATT_ASSERT_STATUS(mjb_filter("A", 1, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, MJB_ENC_UTF_16LE, &result, NULL), MJB_STATUS_OK, "Filter converts output encoding without other changes") ATT_ASSERT(result.transformed, true, "Filter output encoding conversion transformed") ATT_ASSERT(result.output_size, (size_t)2, "Filter output encoding conversion size") ATT_ASSERT((int)memcmp(result.output, "A\0", 2), 0, "Filter output encoding conversion bytes") mjb_result_free(&result); - ATT_ASSERT_STATUS(mjb_filter("A", 1, enc, MJB_FILTER_NORMALIZE, MJB_ENC_UTF_16LE, - &result), + ATT_ASSERT_STATUS(mjb_filter("A", 1, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NORMALIZE, MJB_ENC_UTF_16LE, + &result, NULL), MJB_STATUS_OK, "Filter normalize converts output encoding") ATT_ASSERT(result.transformed, true, "Filter normalize output encoding transformed") ATT_ASSERT(result.output_size, (size_t)2, "Filter normalize output encoding size") @@ -162,15 +168,15 @@ int test_filter(void *arg) { "\xE3\x80\x80"; // U+3000 IDEOGRAPHIC SPACE // clang-format on - ATT_ASSERT_STATUS(mjb_filter(spaces, strlen(spaces), enc, MJB_FILTER_SPACES, enc, - &result), + ATT_ASSERT_STATUS(mjb_filter(spaces, strlen(spaces), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_SPACES, enc, + &result, NULL), MJB_STATUS_OK, "Filter spaces") ATT_ASSERT(result.output, (char *)" ", "Filter spaces output") ATT_ASSERT(result.output_size, 17, "Filter spaces output size") mjb_result_free(&result); - ATT_ASSERT_STATUS(mjb_filter(spaces, strlen(spaces), enc, - (mjb_filter_flags)(MJB_FILTER_NORMALIZE | MJB_FILTER_SPACES), enc, &result), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_filter(spaces, strlen(spaces), enc, MJB_MALFORMED_REPLACE, + (mjb_filter_flags)(MJB_FILTER_NORMALIZE | MJB_FILTER_SPACES), enc, &result, NULL), MJB_STATUS_OK, "Filter spaces and normalize") ATT_ASSERT(result.output, (char *)" ", "Filter spaces and normalize output") ATT_ASSERT(result.output_size, 17, "Filter spaces and normalize output size") @@ -178,32 +184,32 @@ int test_filter(void *arg) { // Test whitespace collapsing with consecutive spaces const char *multiple_spaces = "hello world"; - ATT_ASSERT_STATUS(mjb_filter(multiple_spaces, strlen(multiple_spaces), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "Collapse multiple spaces") + ATT_ASSERT_STATUS(mjb_filter(multiple_spaces, strlen(multiple_spaces), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Collapse multiple spaces") ATT_ASSERT(result.output, (char *)"hello world", "Collapse multiple spaces output") ATT_ASSERT(result.output_size, 11, "Collapse multiple spaces output size") mjb_result_free(&result); // Test whitespace collapsing with tabs and newlines const char *mixed_whitespace = "hello\t\t\n\nworld"; - ATT_ASSERT_STATUS(mjb_filter(mixed_whitespace, strlen(mixed_whitespace), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "Collapse mixed whitespace") + ATT_ASSERT_STATUS(mjb_filter(mixed_whitespace, strlen(mixed_whitespace), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Collapse mixed whitespace") ATT_ASSERT(result.output, (char *)"hello world", "Collapse mixed whitespace output") ATT_ASSERT(result.output_size, 11, "Collapse mixed whitespace output size") mjb_result_free(&result); // Test whitespace collapsing with leading whitespace const char *leading_whitespace = " hello world"; - ATT_ASSERT_STATUS(mjb_filter(leading_whitespace, strlen(leading_whitespace), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "Collapse leading whitespace") + ATT_ASSERT_STATUS(mjb_filter(leading_whitespace, strlen(leading_whitespace), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Collapse leading whitespace") ATT_ASSERT(result.output, (char *)" hello world", "Collapse leading whitespace output") ATT_ASSERT(result.output_size, 12, "Collapse leading whitespace output size") mjb_result_free(&result); // Test whitespace collapsing with trailing whitespace const char *trailing_whitespace = "hello world "; - ATT_ASSERT_STATUS(mjb_filter(trailing_whitespace, strlen(trailing_whitespace), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "Collapse trailing whitespace") + ATT_ASSERT_STATUS(mjb_filter(trailing_whitespace, strlen(trailing_whitespace), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Collapse trailing whitespace") ATT_ASSERT(result.output, (char *)"hello world ", "Collapse trailing whitespace output") ATT_ASSERT(result.output_size, 12, "Collapse trailing whitespace output size") mjb_result_free(&result); @@ -214,7 +220,7 @@ int test_filter(void *arg) { "\xE2\x80\x80\xE2\x80\x81" // U+2000 EN QUAD, U+2001 EM QUAD "world"; ATT_ASSERT_STATUS(mjb_filter(unicode_multiple_spaces, strlen(unicode_multiple_spaces), - enc, (mjb_filter_flags)(MJB_FILTER_SPACES | MJB_FILTER_COLLAPSE_SPACES), enc, &result), + enc, MJB_MALFORMED_REPLACE, (mjb_filter_flags)(MJB_FILTER_SPACES | MJB_FILTER_COLLAPSE_SPACES), enc, &result, NULL), MJB_STATUS_OK, "Collapse Unicode spaces") ATT_ASSERT(result.output, (char *)"hello world", "Collapse Unicode spaces output") ATT_ASSERT(result.output_size, 11, "Collapse Unicode spaces output size") @@ -222,39 +228,39 @@ int test_filter(void *arg) { // Test whitespace collapsing with complex mixed whitespace const char *complex_whitespace = "one \t\n two\r\n\r\nthree four"; - ATT_ASSERT_STATUS(mjb_filter(complex_whitespace, strlen(complex_whitespace), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "Collapse complex whitespace") + ATT_ASSERT_STATUS(mjb_filter(complex_whitespace, strlen(complex_whitespace), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Collapse complex whitespace") ATT_ASSERT(result.output, (char *)"one two three four", "Collapse complex whitespace output") ATT_ASSERT(result.output_size, 18, "Collapse complex whitespace output size") mjb_result_free(&result); // Test whitespace collapsing with only whitespace const char *only_whitespace = " \t\n "; - ATT_ASSERT_STATUS(mjb_filter(only_whitespace, strlen(only_whitespace), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "Collapse only whitespace") + ATT_ASSERT_STATUS(mjb_filter(only_whitespace, strlen(only_whitespace), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "Collapse only whitespace") ATT_ASSERT(result.output, (char *)" ", "Collapse only whitespace output") ATT_ASSERT(result.output_size, 1, "Collapse only whitespace output size") mjb_result_free(&result); // Test no collapsing when there's no consecutive whitespace const char *single_spaces = "hello world test"; - ATT_ASSERT_STATUS(mjb_filter(single_spaces, strlen(single_spaces), enc, - MJB_FILTER_COLLAPSE_SPACES, enc, &result), MJB_STATUS_OK, "No collapse needed") + ATT_ASSERT_STATUS(mjb_filter(single_spaces, strlen(single_spaces), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_COLLAPSE_SPACES, enc, &result, NULL), MJB_STATUS_OK, "No collapse needed") ATT_ASSERT(result.output, (char *)"hello world test", "No collapse needed output") ATT_ASSERT(result.output_size, 16, "No collapse needed output size") mjb_result_free(&result); const char *controls = "\x1\x2\t\n\v\f\r\x1f"; - ATT_ASSERT_STATUS(mjb_filter(controls, strlen(controls), enc, MJB_FILTER_CONTROLS, enc, - &result), MJB_STATUS_OK, "Filter controls") + ATT_ASSERT_STATUS(mjb_filter(controls, strlen(controls), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_CONTROLS, enc, + &result, NULL), MJB_STATUS_OK, "Filter controls") ATT_ASSERT(result.output, (char *)"\t\n\v\f\r", "Filter controls output") ATT_ASSERT(result.output_size, 5, "Filter controls output size") ATT_ASSERT(result.transformed, true, "Filter controls transformed") mjb_result_free(&result); const char *numeric = "1234567890"; - ATT_ASSERT_STATUS(mjb_filter(numeric, strlen(numeric), enc, MJB_FILTER_NUMERIC, enc, - &result), MJB_STATUS_OK, "Filter numeric") + ATT_ASSERT_STATUS(mjb_filter(numeric, strlen(numeric), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NUMERIC, enc, + &result, NULL), MJB_STATUS_OK, "Filter numeric") ATT_ASSERT(result.output, (char *)"1234567890", "Filter numeric output") ATT_ASSERT(result.output_size, 10, "Filter numeric output size") ATT_ASSERT(result.transformed, false, "Filter numeric transformed") @@ -262,16 +268,16 @@ int test_filter(void *arg) { // U+0661 ARABIC-INDIC DIGIT ONE, U+0662 ARABIC-INDIC DIGIT TWO const char *arabic_indic_digit = "\xD9\xA1\xD9\xA2"; - ATT_ASSERT_STATUS(mjb_filter(arabic_indic_digit, strlen(arabic_indic_digit), enc, - MJB_FILTER_NUMERIC, enc, &result), MJB_STATUS_OK, "Filter arabic indic digit") + ATT_ASSERT_STATUS(mjb_filter(arabic_indic_digit, strlen(arabic_indic_digit), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NUMERIC, enc, &result, NULL), MJB_STATUS_OK, "Filter arabic indic digit") ATT_ASSERT(result.output, (char *)"12", "Filter arabic indic digit output") ATT_ASSERT(result.output_size, 2, "Filter arabic indic digit output size") ATT_ASSERT(result.transformed, true, "Filter arabic indic digit transformed") mjb_result_free(&result); const char *ordinary_combining = "Cafe\xCC\x81"; // Cafe + U+0301 COMBINING ACUTE ACCENT - ATT_ASSERT_STATUS(mjb_filter(ordinary_combining, strlen(ordinary_combining), enc, - MJB_FILTER_LIMIT_COMBINING, enc, &result), MJB_STATUS_OK, "Limit combining ordinary accent") + ATT_ASSERT_STATUS(mjb_filter(ordinary_combining, strlen(ordinary_combining), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_LIMIT_COMBINING, enc, &result, NULL), MJB_STATUS_OK, "Limit combining ordinary accent") ATT_ASSERT(result.output, (char *)ordinary_combining, "Limit combining ordinary accent output") ATT_ASSERT(result.output_size, strlen(ordinary_combining), "Limit combining ordinary accent size") @@ -295,8 +301,8 @@ int test_filter(void *arg) { "b"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(stacked_combining, strlen(stacked_combining), enc, - MJB_FILTER_LIMIT_COMBINING, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(stacked_combining, strlen(stacked_combining), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_LIMIT_COMBINING, enc, &result, NULL), MJB_STATUS_OK, "Limit stacked combining marks") ATT_ASSERT(strcmp(result.output, expected_stacked_combining), 0, "Limit stacked combining marks output") @@ -321,7 +327,7 @@ int test_filter(void *arg) { // clang-format on ATT_ASSERT_STATUS(mjb_filter(split_by_removed_control, strlen(split_by_removed_control), - enc, (mjb_filter_flags)(MJB_FILTER_CONTROLS | MJB_FILTER_LIMIT_COMBINING), enc, &result), + enc, MJB_MALFORMED_REPLACE, (mjb_filter_flags)(MJB_FILTER_CONTROLS | MJB_FILTER_LIMIT_COMBINING), enc, &result, NULL), MJB_STATUS_OK, "Limit combining after removed control") ATT_ASSERT(strcmp(result.output, expected_split_by_removed_control), 0, "Limit combining after removed control output") @@ -331,16 +337,16 @@ int test_filter(void *arg) { mjb_result_free(&result); const char *valid_utf8 = "Hello World"; - ATT_ASSERT_STATUS(mjb_filter(valid_utf8, strlen(valid_utf8), enc, MJB_FILTER_NONE, enc, - &result), MJB_STATUS_OK, "UTF8: Valid string") + ATT_ASSERT_STATUS(mjb_filter(valid_utf8, strlen(valid_utf8), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, + &result, NULL), MJB_STATUS_OK, "UTF8: Valid string") ATT_ASSERT(result.output, (char *)"Hello World", "UTF8: Valid string output") ATT_ASSERT(result.output_size, 11, "UTF8: Valid string size") ATT_ASSERT(result.transformed, false, "UTF8: Valid string not transformed") mjb_result_free(&result); const char *valid_multibyte = "Héllo Wörld 世界"; // Latin + CJK - ATT_ASSERT_STATUS(mjb_filter(valid_multibyte, strlen(valid_multibyte), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Valid multibyte") + ATT_ASSERT_STATUS(mjb_filter(valid_multibyte, strlen(valid_multibyte), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Valid multibyte") ATT_ASSERT(strcmp(result.output, valid_multibyte), 0, "UTF8: Valid multibyte unchanged") ATT_ASSERT(result.transformed, false, "UTF8: Valid multibyte not transformed") mjb_result_free(&result); @@ -350,20 +356,51 @@ int test_filter(void *arg) { const char *expected_single = "A\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(single_invalid, strlen(single_invalid), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Single invalid byte") + ATT_ASSERT_STATUS(mjb_filter(single_invalid, strlen(single_invalid), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Single invalid byte") ATT_ASSERT(strcmp(result.output, expected_single), 0, "UTF8: Single invalid byte replaced") ATT_ASSERT(result.output_size, 5, "UTF8: Single invalid byte size") ATT_ASSERT(result.transformed, true, "UTF8: Single invalid byte transformed") mjb_result_free(&result); + mjb_diagnostic diagnostic; + ATT_ASSERT_STATUS(mjb_filter(single_invalid, strlen(single_invalid), enc, MJB_MALFORMED_STOP, + MJB_FILTER_NONE, enc, &result, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "UTF8: Stop policy rejects malformed input") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_OVERLONG_SEQUENCE, + "UTF8: Stop policy reports the malformed kind") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, + "UTF8: Stop policy reports the malformed byte offset") + + ATT_ASSERT_STATUS(mjb_filter(single_invalid, strlen(single_invalid), enc, MJB_MALFORMED_SKIP, + MJB_FILTER_NONE, enc, &result, &diagnostic), + MJB_STATUS_OK, "UTF8: Skip policy filters malformed input") + ATT_ASSERT(result.output_size, (size_t)2, "UTF8: Skip policy output size") + ATT_ASSERT(memcmp(result.output, "AB", 2), 0, "UTF8: Skip policy output") + ATT_ASSERT(result.transformed, true, "UTF8: Skip policy marks the result transformed") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_OVERLONG_SEQUENCE, + "UTF8: Skip policy retains the malformed diagnostic") + mjb_result_free(&result); + + ATT_ASSERT_STATUS(mjb_filter(single_invalid, strlen(single_invalid), enc, + MJB_MALFORMED_REPLACE, MJB_FILTER_NORMALIZE, enc, &result, + &diagnostic), + MJB_STATUS_OK, "UTF8: Replacement policy sanitizes before normalization") + ATT_ASSERT(memcmp(result.output, expected_single, result.output_size), 0, + "UTF8: Normalization preserves the replacement") + ATT_ASSERT(result.transformed, true, + "UTF8: Sanitizing before normalization marks the result transformed") + mjb_result_free(&result); + // clang-format off const char *multiple_invalid = "A\xC0" "B\xC1" "C"; const char *expected_multiple = "A\xEF\xBF\xBD" "B\xEF\xBF\xBD" "C"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(multiple_invalid, strlen(multiple_invalid), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Multiple invalid bytes") + ATT_ASSERT_STATUS(mjb_filter(multiple_invalid, strlen(multiple_invalid), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Multiple invalid bytes") ATT_ASSERT(strcmp(result.output, expected_multiple), 0, "UTF8: Multiple invalid bytes replaced") ATT_ASSERT(result.output_size, 9, "UTF8: Multiple invalid bytes size") ATT_ASSERT(result.transformed, true, "UTF8: Multiple invalid bytes transformed") @@ -371,45 +408,47 @@ int test_filter(void *arg) { // clang-format off const char *invalid_2byte = "A\xC0\xAF" "B"; // Invalid 2-byte sequence - const char *expected_2byte = "A\xEF\xBF\xBD" "B"; + const char *expected_2byte = "A\xEF\xBF\xBD\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(invalid_2byte, strlen(invalid_2byte), enc, MJB_FILTER_NONE, - enc, &result), MJB_STATUS_OK, "UTF8: Invalid 2-byte sequence") + ATT_ASSERT_STATUS(mjb_filter(invalid_2byte, strlen(invalid_2byte), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, + enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid 2-byte sequence") ATT_ASSERT(strcmp(result.output, expected_2byte), 0, - "UTF8: Invalid 2-byte collapsed to one U+FFFD") - ATT_ASSERT(result.output_size, 5, "UTF8: Invalid 2-byte sequence size") + "UTF8: Invalid 2-byte maximal subparts replaced") + ATT_ASSERT(result.output_size, 8, "UTF8: Invalid 2-byte sequence size") ATT_ASSERT(result.transformed, true, "UTF8: Invalid 2-byte transformed") mjb_result_free(&result); // clang-format off const char *invalid_3byte = "A\xE0\x80\x80" "B"; // Invalid 3-byte (overlong) - const char *expected_3byte = "A\xEF\xBF\xBD" "B"; + const char *expected_3byte = + "A\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(invalid_3byte, strlen(invalid_3byte), enc, MJB_FILTER_NONE, - enc, &result), MJB_STATUS_OK, "UTF8: Invalid 3-byte sequence") + ATT_ASSERT_STATUS(mjb_filter(invalid_3byte, strlen(invalid_3byte), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, + enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid 3-byte sequence") ATT_ASSERT(strcmp(result.output, expected_3byte), 0, - "UTF8: Invalid 3-byte collapsed to one U+FFFD") - ATT_ASSERT(result.output_size, 5, "UTF8: Invalid 3-byte sequence size") + "UTF8: Invalid 3-byte maximal subparts replaced") + ATT_ASSERT(result.output_size, 11, "UTF8: Invalid 3-byte sequence size") ATT_ASSERT(result.transformed, true, "UTF8: Invalid 3-byte transformed") mjb_result_free(&result); // clang-format off const char *invalid_4byte = "A\xF5\x80\x80\x80" "B"; // 0xF5 is invalid start - const char *expected_4byte = "A\xEF\xBF\xBD" "B"; + const char *expected_4byte = + "A\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(invalid_4byte, strlen(invalid_4byte), enc, MJB_FILTER_NONE, - enc, &result), MJB_STATUS_OK, "UTF8: Invalid 4-byte sequence") + ATT_ASSERT_STATUS(mjb_filter(invalid_4byte, strlen(invalid_4byte), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, + enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid 4-byte sequence") ATT_ASSERT(strcmp(result.output, expected_4byte), 0, - "UTF8: Invalid 4-byte collapsed to one U+FFFD") - ATT_ASSERT(result.output_size, 5, "UTF8: Invalid 4-byte sequence size") + "UTF8: Invalid 4-byte maximal subparts replaced") + ATT_ASSERT(result.output_size, 14, "UTF8: Invalid 4-byte sequence size") ATT_ASSERT(result.transformed, true, "UTF8: Invalid 4-byte transformed") mjb_result_free(&result); const char *truncated = "Hello\xC2"; // Incomplete 2-byte sequence const char *expected_truncated = "Hello\xEF\xBF\xBD"; - ATT_ASSERT_STATUS(mjb_filter(truncated, 6, enc, MJB_FILTER_NONE, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(truncated, 6, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Truncated sequence") ATT_ASSERT(strcmp(result.output, expected_truncated), 0, "UTF8: Truncated sequence replaced") ATT_ASSERT(result.output_size, 8, "UTF8: Truncated sequence size") @@ -420,8 +459,8 @@ int test_filter(void *arg) { const char *invalid_continuation = "A\x80" "B"; // 0x80 without lead byte const char *expected_continuation = "A\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(invalid_continuation, strlen(invalid_continuation), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Invalid continuation") + ATT_ASSERT_STATUS(mjb_filter(invalid_continuation, strlen(invalid_continuation), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid continuation") ATT_ASSERT(strcmp(result.output, expected_continuation), 0, "UTF8: Invalid continuation replaced") ATT_ASSERT(result.output_size, 5, "UTF8: Invalid continuation size") @@ -429,9 +468,11 @@ int test_filter(void *arg) { mjb_result_free(&result); const char *mixed_valid_invalid = "Hé\xC0llo\xF5\x80\x80\x80 世\xC1界"; - const char *expected_mixed = "Hé\xEF\xBF\xBDllo\xEF\xBF\xBD 世\xEF\xBF\xBD界"; - ATT_ASSERT_STATUS(mjb_filter(mixed_valid_invalid, strlen(mixed_valid_invalid), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Mixed valid/invalid") + const char *expected_mixed = "Hé\xEF\xBF\xBDllo" + "\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD" + " 世\xEF\xBF\xBD界"; + ATT_ASSERT_STATUS(mjb_filter(mixed_valid_invalid, strlen(mixed_valid_invalid), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Mixed valid/invalid") ATT_ASSERT(strcmp(result.output, expected_mixed), 0, "UTF8: Mixed valid/invalid replaced") ATT_ASSERT(result.transformed, true, "UTF8: Mixed valid/invalid transformed") mjb_result_free(&result); @@ -440,8 +481,8 @@ int test_filter(void *arg) { const char *invalid_start = "\xC0" "Hello"; const char *expected_start = "\xEF\xBF\xBD" "Hello"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(invalid_start, strlen(invalid_start), enc, MJB_FILTER_NONE, - enc, &result), MJB_STATUS_OK, "UTF8: Invalid at start") + ATT_ASSERT_STATUS(mjb_filter(invalid_start, strlen(invalid_start), enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, + enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid at start") ATT_ASSERT(strcmp(result.output, expected_start), 0, "UTF8: Invalid at start replaced") ATT_ASSERT(result.output_size, 8, "UTF8: Invalid at start size") ATT_ASSERT(result.transformed, true, "UTF8: Invalid at start transformed") @@ -449,7 +490,7 @@ int test_filter(void *arg) { const char *invalid_end = "Hello\xC0"; const char *expected_end = "Hello\xEF\xBF\xBD"; - ATT_ASSERT_STATUS(mjb_filter(invalid_end, 6, enc, MJB_FILTER_NONE, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(invalid_end, 6, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid at end") ATT_ASSERT(strcmp(result.output, expected_end), 0, "UTF8: Invalid at end replaced") ATT_ASSERT(result.output_size, 8, "UTF8: Invalid at end size") @@ -458,37 +499,38 @@ int test_filter(void *arg) { // clang-format off const char *consecutive_invalid = "A\xC0\xC1\xC2" "B"; // Three consecutive invalid bytes - const char *expected_consecutive = "A\xEF\xBF\xBD" "B"; + const char *expected_consecutive = + "A\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(consecutive_invalid, strlen(consecutive_invalid), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Consecutive invalid bytes") + ATT_ASSERT_STATUS(mjb_filter(consecutive_invalid, strlen(consecutive_invalid), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Consecutive invalid bytes") ATT_ASSERT(strcmp(result.output, expected_consecutive), 0, - "UTF8: Consecutive invalid collapsed") - ATT_ASSERT(result.output_size, 5, "UTF8: Consecutive invalid size") + "UTF8: Consecutive invalid maximal subparts replaced") + ATT_ASSERT(result.output_size, 11, "UTF8: Consecutive invalid size") ATT_ASSERT(result.transformed, true, "UTF8: Consecutive invalid transformed") mjb_result_free(&result); const char *overlong_slash = "\xC0\xAF"; // Overlong encoding of '/' - const char *expected_overlong = "\xEF\xBF\xBD"; - ATT_ASSERT_STATUS(mjb_filter(overlong_slash, 2, enc, MJB_FILTER_NONE, enc, &result), + const char *expected_overlong = "\xEF\xBF\xBD\xEF\xBF\xBD"; + ATT_ASSERT_STATUS(mjb_filter(overlong_slash, 2, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Overlong encoding") ATT_ASSERT(strcmp(result.output, expected_overlong), 0, "UTF8: Overlong encoding replaced") - ATT_ASSERT(result.output_size, 3, "UTF8: Overlong encoding size") + ATT_ASSERT(result.output_size, 6, "UTF8: Overlong encoding size") ATT_ASSERT(result.transformed, true, "UTF8: Overlong encoding transformed") mjb_result_free(&result); const char *only_invalid = "\xC0\xC1\xC2"; - const char *expected_only_invalid = "\xEF\xBF\xBD"; - ATT_ASSERT_STATUS(mjb_filter(only_invalid, 3, enc, MJB_FILTER_NONE, enc, &result), + const char *expected_only_invalid = "\xEF\xBF\xBD\xEF\xBF\xBD\xEF\xBF\xBD"; + ATT_ASSERT_STATUS(mjb_filter(only_invalid, 3, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Only invalid bytes") ATT_ASSERT(strcmp(result.output, expected_only_invalid), 0, "UTF8: Only invalid bytes replaced") - ATT_ASSERT(result.output_size, 3, "UTF8: Only invalid bytes size") + ATT_ASSERT(result.output_size, 9, "UTF8: Only invalid bytes size") ATT_ASSERT(result.transformed, true, "UTF8: Only invalid bytes transformed") mjb_result_free(&result); const char *invalid_then_valid = "\xC0\xE4\xB8\x96"; // Invalid + 世 const char *expected_invalid_valid = "\xEF\xBF\xBD\xE4\xB8\x96"; - ATT_ASSERT_STATUS(mjb_filter(invalid_then_valid, 4, enc, MJB_FILTER_NONE, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(invalid_then_valid, 4, enc, MJB_MALFORMED_REPLACE, MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Invalid then valid") ATT_ASSERT(strcmp(result.output, expected_invalid_valid), 0, "UTF8: Invalid then valid replaced") @@ -500,8 +542,8 @@ int test_filter(void *arg) { const char *missing_continuation_2 = "A\xC2" "B"; // C2 needs continuation const char *expected_missing_2 = "A\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(missing_continuation_2, strlen(missing_continuation_2), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Missing continuation 2-byte") + ATT_ASSERT_STATUS(mjb_filter(missing_continuation_2, strlen(missing_continuation_2), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Missing continuation 2-byte") ATT_ASSERT(strcmp(result.output, expected_missing_2), 0, "UTF8: Missing continuation 2-byte replaced") ATT_ASSERT(result.output_size, 5, "UTF8: Missing continuation 2-byte size") @@ -512,8 +554,8 @@ int test_filter(void *arg) { const char *missing_continuation_3 = "A\xE0\xA0" "B"; // E0 A0 needs another byte const char *expected_missing_3 = "A\xEF\xBF\xBD" "B"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(missing_continuation_3, strlen(missing_continuation_3), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Missing continuation 3-byte") + ATT_ASSERT_STATUS(mjb_filter(missing_continuation_3, strlen(missing_continuation_3), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Missing continuation 3-byte") ATT_ASSERT(strcmp(result.output, expected_missing_3), 0, "UTF8: Missing continuation 3-byte replaced") ATT_ASSERT(result.output_size, 5, "UTF8: Missing continuation 3-byte size") @@ -524,8 +566,8 @@ int test_filter(void *arg) { const char *replacement_then_valid = "\xC0" "Hello"; const char *expected_replacement_valid = "\xEF\xBF\xBD" "Hello"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(replacement_then_valid, strlen(replacement_then_valid), enc, - MJB_FILTER_NONE, enc, &result), + ATT_ASSERT_STATUS(mjb_filter(replacement_then_valid, strlen(replacement_then_valid), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Replacement then valid") ATT_ASSERT(strcmp(result.output, expected_replacement_valid), 0, "UTF8: Replacement then valid text") @@ -535,13 +577,16 @@ int test_filter(void *arg) { // clang-format off const char *separate_invalid = "A\xC0" "B\xE0\x80" "C\xF5\x80" "D"; - const char *expected_separate = "A\xEF\xBF\xBD" "B\xEF\xBF\xBD" "C\xEF\xBF\xBD" "D"; + const char *expected_separate = "A\xEF\xBF\xBD" + "B\xEF\xBF\xBD\xEF\xBF\xBD" + "C\xEF\xBF\xBD\xEF\xBF\xBD" + "D"; // clang-format on - ATT_ASSERT_STATUS(mjb_filter(separate_invalid, strlen(separate_invalid), enc, - MJB_FILTER_NONE, enc, &result), MJB_STATUS_OK, "UTF8: Separate invalid sequences") + ATT_ASSERT_STATUS(mjb_filter(separate_invalid, strlen(separate_invalid), enc, MJB_MALFORMED_REPLACE, + MJB_FILTER_NONE, enc, &result, NULL), MJB_STATUS_OK, "UTF8: Separate invalid sequences") ATT_ASSERT(strcmp(result.output, expected_separate), 0, "UTF8: Separate invalid sequences replaced") - ATT_ASSERT(result.output_size, 13, "UTF8: Separate invalid sequences size") + ATT_ASSERT(result.output_size, 19, "UTF8: Separate invalid sequences size") ATT_ASSERT(result.transformed, true, "UTF8: Separate invalid sequences transformed") mjb_result_free(&result); diff --git a/tests/mojibake.c b/tests/mojibake.c index e32a3e28..3a349ec9 100644 --- a/tests/mojibake.c +++ b/tests/mojibake.c @@ -25,7 +25,8 @@ int test_mojibake(void *arg) { ATT_ASSERT_STATUS(mjb_result_free(NULL), MJB_STATUS_INVALID_ARGUMENT, "Free NULL result") mjb_test_allocator_reset(); - ATT_ASSERT_STATUS(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, &result), + ATT_ASSERT_STATUS(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL), MJB_STATUS_OK, "Allocate result through configured allocator") ATT_ASSERT((int)(mjb_test_allocator_call_count() > 0), true, "Configured allocator receives allocation context") @@ -45,8 +46,8 @@ int test_mojibake(void *arg) { char case_into_output[1]; size_t case_into_size = sizeof(case_into_output); - ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - case_into_output, &case_into_size), + ATT_ASSERT_STATUS(mjb_map_case_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_8, + case_into_output, &case_into_size, NULL), MJB_STATUS_OK, "Caller-buffer case mapping does not allocate") ATT_ASSERT(case_into_size, (size_t)1, "Caller-buffer case mapping output size") ATT_ASSERT((unsigned int)case_into_output[0], (unsigned int)'A', @@ -55,8 +56,9 @@ int test_mojibake(void *arg) { char filter_into_output[3]; size_t filter_into_size = sizeof(filter_into_output); - ATT_ASSERT_STATUS(mjb_filter_into("a", 1, MJB_ENC_UTF_8, MJB_FILTER_NONE, MJB_ENC_UTF_8, - filter_into_output, &filter_into_size), + ATT_ASSERT_STATUS(mjb_filter_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_NONE, MJB_ENC_UTF_8, filter_into_output, &filter_into_size, + NULL), MJB_STATUS_OK, "Caller-buffer filtering does not allocate") ATT_ASSERT(filter_into_size, (size_t)1, "Caller-buffer filtering output size") ATT_ASSERT((unsigned int)filter_into_output[0], (unsigned int)'a', @@ -65,9 +67,9 @@ int test_mojibake(void *arg) { char normalize_into_output[3]; size_t normalize_into_size = sizeof(normalize_into_output); - ATT_ASSERT_STATUS(mjb_normalize_into("\xC3\xA9", 2, MJB_ENC_UTF_8, + ATT_ASSERT_STATUS(mjb_normalize_into("\xC3\xA9", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, normalize_into_output, - &normalize_into_size), + &normalize_into_size, NULL), MJB_STATUS_OK, "Caller-buffer decomposition does not allocate") ATT_ASSERT(normalize_into_size, (size_t)3, "Caller-buffer decomposition output size") ATT_ASSERT((int)memcmp(normalize_into_output, "e\xCC\x81", 3), 0, @@ -75,36 +77,37 @@ int test_mojibake(void *arg) { char normalize_utf16_output[2]; normalize_into_size = sizeof(normalize_utf16_output); - ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_16LE, normalize_utf16_output, &normalize_into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_16LE, normalize_utf16_output, &normalize_into_size, NULL), MJB_STATUS_OK, "Caller-buffer normalized encoding conversion does not allocate") ATT_ASSERT(normalize_into_size, (size_t)2, "Caller-buffer normalized encoding conversion output size") normalize_into_size = 0; - ATT_ASSERT_STATUS(mjb_normalize_into("e\xCC\x81", 3, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &normalize_into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("e\xCC\x81", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &normalize_into_size, NULL), MJB_STATUS_NO_MEMORY, "Caller-buffer composition handles temporary allocation failure") ATT_ASSERT(normalize_into_size, (size_t)0, "Caller-buffer composition clears size after allocation failure") filter_into_size = sizeof(filter_into_output); - ATT_ASSERT_STATUS(mjb_filter_into("e\xCC\x81", 3, MJB_ENC_UTF_8, MJB_FILTER_NORMALIZE, - MJB_ENC_UTF_8, filter_into_output, &filter_into_size), + ATT_ASSERT_STATUS(mjb_filter_into("e\xCC\x81", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_NORMALIZE, MJB_ENC_UTF_8, filter_into_output, + &filter_into_size, NULL), MJB_STATUS_NO_MEMORY, "Caller-buffer normalization handles temporary allocation failure") size_t nfkc_casefold_into_size = 0; - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into("a", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_8, NULL, - &nfkc_casefold_into_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, NULL, + &nfkc_casefold_into_size, NULL), MJB_STATUS_NO_MEMORY, "Caller-buffer NFKC casefold handles temporary allocation failure") ATT_ASSERT(nfkc_casefold_into_size, (size_t)0, "Caller-buffer NFKC casefold clears size after allocation failure") #if MJB_FEATURE_COLLATION size_t collation_key_into_size = 0; - ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, + ATT_ASSERT_STATUS(mjb_collation_key_into("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, - &collation_key_into_size), + &collation_key_into_size, NULL), MJB_STATUS_NO_MEMORY, "Caller-buffer collation key handles temporary allocation failure") ATT_ASSERT(collation_key_into_size, (size_t)0, "Caller-buffer collation key clears size after allocation failure") @@ -119,19 +122,20 @@ int test_mojibake(void *arg) { "Caller-buffer skeleton clears size after allocation failure") #endif - ATT_ASSERT_STATUS(mjb_convert_encoding("a", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, &result), + ATT_ASSERT_STATUS(mjb_convert_encoding("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL), MJB_STATUS_NO_MEMORY, "Encoding conversion handles allocation failure") - ATT_ASSERT_STATUS(mjb_filter("a", 1, MJB_ENC_UTF_8, MJB_FILTER_NONE, MJB_ENC_UTF_8, - &result), + ATT_ASSERT_STATUS(mjb_filter("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_FILTER_NONE, + MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_NO_MEMORY, "Filter handles allocation failure") - ATT_ASSERT_STATUS(mjb_normalize("e\xCC\x81", 3, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_normalize("e\xCC\x81", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_NO_MEMORY, "Normalization handles allocation failure") - ATT_ASSERT_STATUS(mjb_map_case("a", 1, MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_map_case("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_NO_MEMORY, "Case conversion handles allocation failure") #if MJB_FEATURE_COLLATION - ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, &result), + ATT_ASSERT_STATUS(mjb_collation_key("a", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, + MJB_COLLATION_TERTIARY, &result, NULL), MJB_STATUS_NO_MEMORY, "Collation key handles allocation failure") int order; ATT_ASSERT_STATUS(mjb_collation_compare("a", 1, MJB_ENC_UTF_8, "b", 1, MJB_ENC_UTF_8, @@ -148,14 +152,16 @@ int test_mojibake(void *arg) { mjb_test_allocator_reset(); test_set_failing_allocator(1); - ATT_ASSERT_STATUS(mjb_convert_encoding("ab", 2, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, - &result), + ATT_ASSERT_STATUS(mjb_convert_encoding("ab", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL), MJB_STATUS_NO_MEMORY, "Encoding conversion handles reallocation failure") mjb_test_allocator_reset(); ATT_ASSERT(mjb_status_message(MJB_STATUS_OK), "The operation completed successfully", "Status message returns OK") + ATT_ASSERT(mjb_status_message(MJB_STATUS_END_OF_INPUT), "The end of the input was reached", + "Status message returns end of input") ATT_ASSERT(mjb_status_message((mjb_status)100), "The status code is unknown", "Status message returns unknown error for invalid status") diff --git a/tests/next.c b/tests/next.c index a65da0ca..d0404389 100644 --- a/tests/next.c +++ b/tests/next.c @@ -21,18 +21,50 @@ static bool stop_next_character(mjb_character *character, mjb_character_position } int test_next(void *arg) { - ATT_ASSERT_STATUS(mjb_for_each_codepoint(NULL, 1, MJB_ENC_UTF_8, next_character), + ATT_ASSERT_STATUS(mjb_for_each_codepoint(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + next_character, NULL), MJB_STATUS_INVALID_ARGUMENT, "NULL next buffer") - ATT_ASSERT_STATUS(mjb_for_each_codepoint("A", 1, MJB_ENC_UTF_8, NULL), + ATT_ASSERT_STATUS(mjb_for_each_codepoint("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, NULL, + NULL), MJB_STATUS_INVALID_ARGUMENT, "NULL next callback") - ATT_ASSERT_STATUS(mjb_for_each_codepoint("Hèllò", 7, MJB_ENC_UTF_8, next_character), + ATT_ASSERT_STATUS(mjb_for_each_codepoint("Hèllò", 7, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + next_character, NULL), MJB_STATUS_OK, "Next character") ATT_ASSERT(mjb_test_count, 5, "mjb_for_each_codepoint") mjb_test_count = 0; - ATT_ASSERT_STATUS(mjb_for_each_codepoint("A", 1, MJB_ENC_UTF_8, stop_next_character), + ATT_ASSERT_STATUS(mjb_for_each_codepoint("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + stop_next_character, NULL), MJB_STATUS_CALLBACK_STOPPED, "Next character callback stopped") ATT_ASSERT(mjb_test_count, 1, "mjb_for_each_codepoint stopped after callback") + mjb_test_count = 0; + ATT_ASSERT_STATUS(mjb_for_each_codepoint(NULL, 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + next_character, NULL), + MJB_STATUS_OK, "Empty codepoint iteration") + ATT_ASSERT(mjb_test_count, (size_t)0, "Empty codepoint iteration has no callbacks") + + const char malformed[] = { 'A', '\x80', 'B' }; + mjb_diagnostic diagnostic; + + ATT_ASSERT_STATUS(mjb_for_each_codepoint(malformed, sizeof(malformed), MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, next_character, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Codepoint iteration stop policy") + ATT_ASSERT(mjb_test_count, (size_t)0, "Stop policy validates before invoking callbacks") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + "Codepoint iteration stop diagnostic") + + ATT_ASSERT_STATUS(mjb_for_each_codepoint(malformed, sizeof(malformed), MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, next_character, &diagnostic), + MJB_STATUS_OK, "Codepoint iteration replace policy") + ATT_ASSERT(mjb_test_count, (size_t)3, "Codepoint iteration includes replacement") + + mjb_test_count = 0; + ATT_ASSERT_STATUS(mjb_for_each_codepoint(malformed, sizeof(malformed), MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, next_character, &diagnostic), + MJB_STATUS_OK, "Codepoint iteration skip policy") + ATT_ASSERT(mjb_test_count, (size_t)2, "Codepoint iteration skips malformed input") + return 0; } diff --git a/tests/normalization.c b/tests/normalization.c index 7e9135ca..64706ccc 100644 --- a/tests/normalization.c +++ b/tests/normalization.c @@ -47,8 +47,8 @@ static int check_normalization(char *source, size_t source_size, char *normalize char test_name[128]; MJB_TEST_COVERAGE(mjb_normalize); - mjb_status status = mjb_normalize(source, source_size, MJB_ENC_UTF_8, form, MJB_ENC_UTF_8, - &result); + mjb_status status = mjb_normalize(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, form, + MJB_ENC_UTF_8, &result, NULL); if(status != MJB_STATUS_OK) { snprintf(test_name, 128, "#%u %s", current_line, step); @@ -69,15 +69,15 @@ static int check_normalization(char *source, size_t source_size, char *normalize printf("\n%s: source:", test_name); print_status = mjb_for_each_codepoint(source, source_size, MJB_ENC_UTF_8, - next_codepoint); + MJB_MALFORMED_STOP, next_codepoint, NULL); (void)print_status; // we can ignore this. printf("\nExpected: "); print_status = mjb_for_each_codepoint(normalized, normalized_size, MJB_ENC_UTF_8, - next_codepoint); + MJB_MALFORMED_STOP, next_codepoint, NULL); (void)print_status; // we can ignore this. printf("\nGot: "); print_status = mjb_for_each_codepoint(result.output, result.output_size, MJB_ENC_UTF_8, - next_codepoint); + MJB_MALFORMED_STOP, next_codepoint, NULL); (void)print_status; // we can ignore this. puts(""); } @@ -112,8 +112,8 @@ static int check_normalization(char *source, size_t source_size, char *normalize MJB_TEST_COVERAGE(mjb_normalize_into); size_t required = 0; - ATT_ASSERT_STATUS(mjb_normalize_into(source, source_size, MJB_ENC_UTF_8, form, MJB_ENC_UTF_8, - NULL, &required), + ATT_ASSERT_STATUS(mjb_normalize_into(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, form, MJB_ENC_UTF_8, + NULL, &required, NULL), MJB_STATUS_OK, test_name) ATT_ASSERT(required, normalized_size, test_name) @@ -126,8 +126,8 @@ static int check_normalization(char *source, size_t source_size, char *normalize memset(into_output, '#', sizeof(into_output)); size_t output_size = required; - ATT_ASSERT_STATUS(mjb_normalize_into(source, source_size, MJB_ENC_UTF_8, form, MJB_ENC_UTF_8, - into_output, &output_size), + ATT_ASSERT_STATUS(mjb_normalize_into(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, form, MJB_ENC_UTF_8, + into_output, &output_size, NULL), MJB_STATUS_OK, test_name) ATT_ASSERT(output_size, normalized_size, test_name) ATT_ASSERT((int)memcmp(into_output, normalized, normalized_size), 0, test_name) @@ -141,8 +141,8 @@ static void check_nfkc_casefold_into(const char *source, size_t source_size, size_t required = 0; MJB_TEST_COVERAGE(mjb_nfkc_casefold_into); - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(source, source_size, MJB_ENC_UTF_8, output_encoding, - NULL, &required), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, output_encoding, + NULL, &required, NULL), MJB_STATUS_OK, name) ATT_ASSERT(required, allocated->output_size, name) @@ -155,8 +155,8 @@ static void check_nfkc_casefold_into(const char *source, size_t source_size, memset(output, '#', sizeof(output)); size_t output_size = required; - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(source, source_size, MJB_ENC_UTF_8, output_encoding, - output, &output_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, output_encoding, + output, &output_size, NULL), MJB_STATUS_OK, name) ATT_ASSERT(output_size, allocated->output_size, name) ATT_ASSERT((int)memcmp(output, allocated->output, output_size), 0, name) @@ -168,7 +168,7 @@ static void check_nfkc_casefold(const char *source, size_t source_size, const ch mjb_result result = { NULL, 0, false }; MJB_TEST_COVERAGE(mjb_nfkc_casefold); - ATT_ASSERT_STATUS(mjb_nfkc_casefold(source, source_size, MJB_ENC_UTF_8, MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_nfkc_casefold(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_OK, name) ATT_ASSERT(result.output_size, expected_size, name) ATT_ASSERT((int)memcmp(result.output, expected, expected_size), 0, name) @@ -180,45 +180,45 @@ static void check_nfkc_casefold(const char *source, size_t source_size, const ch static void test_nfkc_casefold(void) { mjb_result result = { NULL, 0, false }; - ATT_ASSERT_STATUS(mjb_nfkc_casefold(NULL, 1, MJB_ENC_UTF_8, MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_nfkc_casefold(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_INVALID_ARGUMENT, "NFKC casefold rejects NULL buffer") - ATT_ASSERT_STATUS(mjb_nfkc_casefold("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_8, NULL), + ATT_ASSERT_STATUS(mjb_nfkc_casefold("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "NFKC casefold rejects NULL result") - ATT_ASSERT_STATUS(mjb_nfkc_casefold("", 0, MJB_ENC_UTF_8, MJB_ENC_UTF_8, &result), + ATT_ASSERT_STATUS(mjb_nfkc_casefold("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &result, NULL), MJB_STATUS_OK, "NFKC casefold accepts empty input") ATT_ASSERT(result.transformed, false, "NFKC casefold empty input is borrowed") size_t into_size = 9; - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(NULL, 1, MJB_ENC_UTF_8, MJB_ENC_UTF_8, NULL, - &into_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, NULL, + &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "NFKC casefold into rejects NULL buffer") ATT_ASSERT(into_size, (size_t)0, "NFKC casefold into clears size after invalid input") - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_8, NULL, NULL), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, NULL, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "NFKC casefold into rejects NULL size") - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into("", 0, MJB_ENC_UTF_8, MJB_ENC_UTF_8, NULL, - &into_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, NULL, + &into_size, NULL), MJB_STATUS_OK, "NFKC casefold into measures empty input") ATT_ASSERT(into_size, (size_t)0, "NFKC casefold into empty size") const char *into_input = "Stra\xC3\x9F" "e\xC2\xAD"; - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(into_input, 9, MJB_ENC_UTF_8, MJB_ENC_UTF_8, NULL, - &into_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(into_input, 9, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, NULL, + &into_size, NULL), MJB_STATUS_OK, "NFKC casefold into queries required size") ATT_ASSERT(into_size, (size_t)7, "NFKC casefold into required size") char into_output[8] = { '#', '#', '#', '#', '#', '#', '#', '#' }; into_size = 6; - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(into_input, 9, MJB_ENC_UTF_8, MJB_ENC_UTF_8, - into_output, &into_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(into_input, 9, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, + into_output, &into_size, NULL), MJB_STATUS_OUTPUT_TOO_SMALL, "NFKC casefold into reports a small output buffer") ATT_ASSERT(into_size, (size_t)7, "NFKC casefold into preserves required size") ATT_ASSERT((int)memcmp(into_output, "########", sizeof(into_output)), 0, "NFKC casefold into leaves a small buffer untouched") into_size = 7; - ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(into_input, 9, MJB_ENC_UTF_8, MJB_ENC_UTF_8, - into_output, &into_size), + ATT_ASSERT_STATUS(mjb_nfkc_casefold_into(into_input, 9, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, + into_output, &into_size, NULL), MJB_STATUS_OK, "NFKC casefold into writes into exact capacity") ATT_ASSERT(into_size, (size_t)7, "NFKC casefold into written size") ATT_ASSERT((int)memcmp(into_output, "strasse", 7), 0, "NFKC casefold into output") @@ -233,7 +233,7 @@ static void test_nfkc_casefold(void) { check_nfkc_casefold("\xE2\x84\xAA", 3, "k", 1, "NFKC casefold applies compatibility case mapping"); - ATT_ASSERT_STATUS(mjb_nfkc_casefold("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, &result), + ATT_ASSERT_STATUS(mjb_nfkc_casefold("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, &result, NULL), MJB_STATUS_OK, "NFKC casefold supports UTF-16 output") ATT_ASSERT(result.output_size, (size_t)2, "NFKC casefold UTF-16 output size") ATT_ASSERT((int)memcmp(result.output, "a\0", 2), 0, "NFKC casefold UTF-16 output bytes") @@ -300,8 +300,8 @@ static void test_nfkc_casefold_file(void) { size_t mapping_size = get_string_from_codepoints(second_semicolon + 1, sizeof(expected_mapping), expected_mapping); mjb_result expected; - ATT_ASSERT_STATUS(mjb_normalize(expected_mapping, mapping_size, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &expected), + ATT_ASSERT_STATUS(mjb_normalize(expected_mapping, mapping_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &expected, NULL), MJB_STATUS_OK, "Normalize expected NFKC casefold mapping") for(unsigned int codepoint = start; codepoint <= end; ++codepoint) { @@ -312,8 +312,8 @@ static void test_nfkc_casefold_file(void) { char test_name[128]; snprintf(test_name, sizeof(test_name), "NFKC_CF #%u U+%04X", current_line, codepoint); - ATT_ASSERT_STATUS(mjb_nfkc_casefold(source, source_size, MJB_ENC_UTF_8, MJB_ENC_UTF_8, - &actual), + ATT_ASSERT_STATUS(mjb_nfkc_casefold(source, source_size, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_8, + &actual, NULL), MJB_STATUS_OK, test_name) ATT_ASSERT(actual.output_size, expected.output_size, test_name) ATT_ASSERT((int)memcmp(actual.output, expected.output, expected.output_size), 0, @@ -331,6 +331,75 @@ static void test_nfkc_casefold_file(void) { fclose(file); } +static void test_normalization_malformed_policies(void) { + const char malformed[] = "A\x80" + "e\xCC\x81"; + mjb_diagnostic diagnostic; + mjb_result result = { NULL, 0, false }; + + ATT_ASSERT_STATUS(mjb_normalize(malformed, sizeof(malformed) - 1, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, + &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Normalize stop policy rejects malformed input") + ATT_ASSERT((unsigned int)diagnostic.error, + (unsigned int)MJB_TEXT_ERROR_UNEXPECTED_CONTINUATION, + "Normalize reports malformed input kind") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Normalize reports malformed byte offset") + ATT_ASSERT(diagnostic.byte_length, (size_t)1, "Normalize reports malformed byte length") + + ATT_ASSERT_STATUS(mjb_normalize(malformed, sizeof(malformed) - 1, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, + &diagnostic), + MJB_STATUS_OK, "Normalize replace policy accepts malformed input") + ATT_ASSERT(result.output_size, (size_t)6, "Normalize replacement output size") + ATT_ASSERT((int)memcmp(result.output, "A\xEF\xBF\xBD\xC3\xA9", result.output_size), 0, + "Normalize replacement output") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, + "Normalize replacement retains malformed diagnostic") + ATT_ASSERT_STATUS(mjb_result_free(&result), MJB_STATUS_OK, + "Free normalization replacement result") + + ATT_ASSERT_STATUS(mjb_normalize(malformed, sizeof(malformed) - 1, MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, + &diagnostic), + MJB_STATUS_OK, "Normalize skip policy accepts malformed input") + ATT_ASSERT(result.output_size, (size_t)3, "Normalize skip output size") + ATT_ASSERT((int)memcmp(result.output, "A\xC3\xA9", result.output_size), 0, + "Normalize skip output") + ATT_ASSERT_STATUS(mjb_result_free(&result), MJB_STATUS_OK, "Free normalization skip result") + + size_t output_size = 0; + ATT_ASSERT_STATUS(mjb_normalize_into(malformed, sizeof(malformed) - 1, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, + &output_size, &diagnostic), + MJB_STATUS_OK, "Normalize into measures replacement output") + ATT_ASSERT(output_size, (size_t)6, "Normalize into replacement size") + + const char malformed_casefold[] = "A\x80" + "B"; + ATT_ASSERT_STATUS(mjb_nfkc_casefold(malformed_casefold, sizeof(malformed_casefold) - 1, + MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE, MJB_ENC_UTF_8, &result, + &diagnostic), + MJB_STATUS_OK, "NFKC casefold replace policy accepts malformed input") + ATT_ASSERT(result.output_size, (size_t)5, "NFKC casefold replacement output size") + ATT_ASSERT((int)memcmp(result.output, "a\xEF\xBF\xBD" + "b", + result.output_size), + 0, "NFKC casefold replacement output") + ATT_ASSERT_STATUS(mjb_result_free(&result), MJB_STATUS_OK, + "Free NFKC casefold replacement result") + + ATT_ASSERT_STATUS(mjb_nfkc_casefold(malformed_casefold, sizeof(malformed_casefold) - 1, + MJB_ENC_UTF_8, MJB_MALFORMED_SKIP, MJB_ENC_UTF_8, &result, + &diagnostic), + MJB_STATUS_OK, "NFKC casefold skip policy accepts malformed input") + ATT_ASSERT(result.output_size, (size_t)2, "NFKC casefold skip output size") + ATT_ASSERT((int)memcmp(result.output, "ab", result.output_size), 0, + "NFKC casefold skip output") + ATT_ASSERT_STATUS(mjb_result_free(&result), MJB_STATUS_OK, + "Free NFKC casefold skip result") +} + /** * Run utils/generate/unicode-data/UCD/NormalizationTest.txt tests */ @@ -342,12 +411,13 @@ int test_normalization(void *arg) { test_nfkc_casefold(); test_nfkc_casefold_file(); + test_normalization_malformed_policies(); - ATT_ASSERT_STATUS(mjb_normalize(NULL, 1, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &guard_result), + ATT_ASSERT_STATUS(mjb_normalize(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, + &guard_result, NULL), MJB_STATUS_INVALID_ARGUMENT, "Normalize rejects NULL buffer") - ATT_ASSERT_STATUS(mjb_normalize("", 0, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - NULL), + ATT_ASSERT_STATUS(mjb_normalize("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, + NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Normalize rejects NULL result") char noncharacter_source[] = "\xF1\x8F\xBF\xBE>\xCC\xB8"; @@ -359,8 +429,8 @@ int test_normalization(void *arg) { check_normalization(hangul_source, 13, hangul_normalized, 7, MJB_NORMALIZATION_NFC, 0, "NFC composes after Hangul compaction"); - ATT_ASSERT_STATUS(mjb_normalize("A", 1, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_16LE, - &guard_result), + ATT_ASSERT_STATUS(mjb_normalize("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_16LE, + &guard_result, NULL), MJB_STATUS_OK, "Normalize converts output encoding for already-normalized input") ATT_ASSERT(guard_result.transformed, true, "Normalize converted already-normalized input transformed") @@ -373,45 +443,45 @@ int test_normalization(void *arg) { } size_t into_size = 9; - ATT_ASSERT_STATUS(mjb_normalize_into(NULL, 1, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_8, NULL, &into_size, NULL), MJB_STATUS_INVALID_ARGUMENT, "Normalize into rejects NULL buffer") ATT_ASSERT(into_size, (size_t)0, "Normalize into clears size after invalid input") - ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, NULL, NULL), + ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_8, NULL, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Normalize into rejects NULL size") into_size = 9; - ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, (mjb_normalization)99, - MJB_ENC_UTF_8, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, (mjb_normalization)99, + MJB_ENC_UTF_8, NULL, &into_size, NULL), MJB_STATUS_INVALID_FORM, "Normalize into rejects invalid form") ATT_ASSERT(into_size, (size_t)0, "Normalize into clears size after invalid form") into_size = 9; - ATT_ASSERT_STATUS(mjb_normalize_into("", 0, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_8, NULL, &into_size, NULL), MJB_STATUS_OK, "Normalize into measures empty input") ATT_ASSERT(into_size, (size_t)0, "Normalize into empty size") const char *into_input = "Cafe\xCC\x81"; into_size = 0; - ATT_ASSERT_STATUS(mjb_normalize_into(into_input, 6, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into(into_input, 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &into_size, NULL), MJB_STATUS_OK, "Normalize into queries required size") ATT_ASSERT(into_size, (size_t)5, "Normalize into required size") char into_output[6] = { '#', '#', '#', '#', '#', '#' }; into_size = 4; - ATT_ASSERT_STATUS(mjb_normalize_into(into_input, 6, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, into_output, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into(into_input, 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, into_output, &into_size, NULL), MJB_STATUS_OUTPUT_TOO_SMALL, "Normalize into reports a small output buffer") ATT_ASSERT(into_size, (size_t)5, "Normalize into preserves required size") ATT_ASSERT((int)memcmp(into_output, "######", sizeof(into_output)), 0, "Normalize into leaves a small buffer untouched") into_size = 5; - ATT_ASSERT_STATUS(mjb_normalize_into(into_input, 6, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, into_output, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into(into_input, 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, into_output, &into_size, NULL), MJB_STATUS_OK, "Normalize into writes into exact capacity") ATT_ASSERT(into_size, (size_t)5, "Normalize into written size") ATT_ASSERT((int)memcmp(into_output, "Caf\xC3\xA9", 5), 0, "Normalize into output") @@ -419,12 +489,12 @@ int test_normalization(void *arg) { char nfd_output[4] = { '#', '#', '#', '#' }; into_size = 0; - ATT_ASSERT_STATUS(mjb_normalize_into("\xC3\xA9", 2, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, NULL, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("\xC3\xA9", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, NULL, &into_size, NULL), MJB_STATUS_OK, "Normalize into measures decomposed output") ATT_ASSERT(into_size, (size_t)3, "Normalize into decomposed required size") - ATT_ASSERT_STATUS(mjb_normalize_into("\xC3\xA9", 2, MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, nfd_output, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("\xC3\xA9", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFD, MJB_ENC_UTF_8, nfd_output, &into_size, NULL), MJB_STATUS_OK, "Normalize into writes decomposed output") ATT_ASSERT((int)memcmp(nfd_output, "e\xCC\x81", 3), 0, "Normalize into decomposed output bytes") @@ -432,8 +502,8 @@ int test_normalization(void *arg) { char utf16_output[3] = { '#', '#', '#' }; into_size = sizeof(utf16_output) - 1; - ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_16LE, utf16_output, &into_size), + ATT_ASSERT_STATUS(mjb_normalize_into("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, + MJB_ENC_UTF_16LE, utf16_output, &into_size, NULL), MJB_STATUS_OK, "Normalize into converts already-normalized output encoding") ATT_ASSERT(into_size, (size_t)2, "Normalize into converted output size") ATT_ASSERT((int)memcmp(utf16_output, "A\0", 2), 0, diff --git a/tests/quick-check.c b/tests/quick-check.c index ced9c92a..fbb13bae 100644 --- a/tests/quick-check.c +++ b/tests/quick-check.c @@ -44,6 +44,11 @@ int test_quick_check(void *arg) { assert_quick_check((const char *)malformed_utf8, sizeof(malformed_utf8), enc, MJB_NORMALIZATION_NFC, MJB_STATUS_MALFORMED_INPUT, MJB_QC_NO, "Malformed UTF-8 quick check"); + const unsigned char non_normalized_then_malformed[] = { 0xC3, 0xA9, 0x80 }; + assert_quick_check((const char *)non_normalized_then_malformed, + sizeof(non_normalized_then_malformed), enc, MJB_NORMALIZATION_NFD, + MJB_STATUS_MALFORMED_INPUT, MJB_QC_NO, + "Quick check validates beyond an early normalization result"); assert_quick_check("", 0, enc, MJB_NORMALIZATION_NFC, MJB_STATUS_OK, MJB_QC_YES, "Empty string is NFC normalized"); diff --git a/tests/segmentation.c b/tests/segmentation.c index c1d65325..d6926a99 100644 --- a/tests/segmentation.c +++ b/tests/segmentation.c @@ -184,88 +184,101 @@ static void test_grapheme_count(void) { size_t count = 12345; // Argument validation - ATT_ASSERT_STATUS(mjb_grapheme_count("A", 1, MJB_ENC_UTF_8, NULL), + ATT_ASSERT_STATUS(mjb_grapheme_count("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Count rejects NULL count") - ATT_ASSERT_STATUS(mjb_grapheme_count(NULL, 1, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_grapheme_count(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_INVALID_ARGUMENT, "Count rejects NULL buffer") ATT_ASSERT(count, (size_t)0, "Count is zero after NULL buffer") count = 12345; - ATT_ASSERT_STATUS(mjb_grapheme_count("A", 1, MJB_ENC_UNKNOWN, &count), + ATT_ASSERT_STATUS(mjb_grapheme_count("A", 1, MJB_ENC_UNKNOWN, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_INVALID_ENCODING, "Count rejects invalid encoding") ATT_ASSERT(count, (size_t)0, "Count is zero after invalid encoding") // Empty input is valid and counts zero clusters - ATT_ASSERT_STATUS(mjb_grapheme_count("", 0, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: empty string status") ATT_ASSERT(count, (size_t)0, "Count: empty string") - ATT_ASSERT_STATUS(mjb_grapheme_count(NULL, 0, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count(NULL, 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: NULL buffer with zero length status") // ASCII: each byte is one grapheme cluster - ATT_ASSERT_STATUS(mjb_grapheme_count("ABC", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count("ABC", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: ABC status") ATT_ASSERT(count, (size_t)3, "Count: ABC") // "aé" = 0x61 0xC3 0xA9 = 3 bytes, 2 grapheme clusters - ATT_ASSERT_STATUS(mjb_grapheme_count("a\xC3\xA9", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count("a\xC3\xA9", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: aé status") ATT_ASSERT(count, (size_t)2, "Count: aé") // a + combining acute accent = one user-perceived character - ATT_ASSERT_STATUS(mjb_grapheme_count("a\xCC\x81", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count("a\xCC\x81", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: a + combining acute status") ATT_ASSERT(count, (size_t)1, "Count: a + combining acute") // Flag emoji 🇺🇸 = two RI codepoints, one grapheme cluster; two flags do not pair across - ATT_ASSERT_STATUS(mjb_grapheme_count("\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8", 8, MJB_ENC_UTF_8, - &count), MJB_STATUS_OK, "Count: flag emoji status") + ATT_ASSERT_STATUS(mjb_grapheme_count("\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8", 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL), MJB_STATUS_OK, "Count: flag emoji status") ATT_ASSERT(count, (size_t)1, "Count: flag emoji") ATT_ASSERT_STATUS(mjb_grapheme_count("\xF0\x9F\x87\xAE\xF0\x9F\x87\xB9" - "\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8", 16, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + "\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8", 16, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: two flags status") ATT_ASSERT(count, (size_t)2, "Count: two flags") // Family ZWJ sequence 👨‍👩‍👦 = one grapheme cluster ATT_ASSERT_STATUS(mjb_grapheme_count("\xF0\x9F\x91\xA8\xE2\x80\x8D\xF0\x9F\x91\xA9" - "\xE2\x80\x8D\xF0\x9F\x91\xA6", 18, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + "\xE2\x80\x8D\xF0\x9F\x91\xA6", 18, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: family ZWJ status") ATT_ASSERT(count, (size_t)1, "Count: family ZWJ sequence") // Hangul jamo L+V+T = one syllable cluster - ATT_ASSERT_STATUS(mjb_grapheme_count("\xE1\x84\x80\xE1\x85\xA1\xE1\x86\xA8", 9, MJB_ENC_UTF_8, - &count), MJB_STATUS_OK, "Count: Hangul jamo status") + ATT_ASSERT_STATUS(mjb_grapheme_count("\xE1\x84\x80\xE1\x85\xA1\xE1\x86\xA8", 9, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL), MJB_STATUS_OK, "Count: Hangul jamo status") ATT_ASSERT(count, (size_t)1, "Count: Hangul jamo LVT syllable") // CRLF is a single cluster (GB3) - ATT_ASSERT_STATUS(mjb_grapheme_count("a\r\nb", 4, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count("a\r\nb", 4, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: CRLF status") ATT_ASSERT(count, (size_t)3, "Count: a CRLF b") - // Malformed byte mid-string counts as one replacement cluster - ATT_ASSERT_STATUS(mjb_grapheme_count("a\x80z", 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, - "Count: malformed mid-string status") - ATT_ASSERT(count, (size_t)3, "Count: malformed mid-string") - - // Incomplete trailing sequence does not add a cluster - ATT_ASSERT_STATUS(mjb_grapheme_count("a\xC3", 2, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, - "Count: incomplete tail status") - ATT_ASSERT(count, (size_t)1, "Count: incomplete tail") + mjb_diagnostic diagnostic; + ATT_ASSERT_STATUS(mjb_grapheme_count("a\x80z", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Count rejects malformed input") + ATT_ASSERT(count, (size_t)0, "Count is zero after malformed input") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Grapheme count malformed offset") + + ATT_ASSERT_STATUS(mjb_grapheme_count("a\x80z", 3, MJB_ENC_UTF_8, + MJB_MALFORMED_REPLACE, &count, &diagnostic), + MJB_STATUS_OK, "Grapheme count replaces malformed input") + ATT_ASSERT(count, (size_t)3, "Replacement is a grapheme") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, + "Grapheme replacement retains malformed offset") + + ATT_ASSERT_STATUS(mjb_grapheme_count("a\x80z", 3, MJB_ENC_UTF_8, + MJB_MALFORMED_SKIP, &count, &diagnostic), + MJB_STATUS_OK, "Grapheme count skips malformed input") + ATT_ASSERT(count, (size_t)2, "Skipped malformed input is not a grapheme") + + ATT_ASSERT_STATUS(mjb_grapheme_count("a\xC3", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), + MJB_STATUS_MALFORMED_INPUT, "Count rejects an incomplete tail") + ATT_ASSERT(count, (size_t)0, "Count is zero after an incomplete tail") // Explicit lengths include embedded U+0000 codepoints const char embedded_nul[] = { 'A', '\0', 'B' }; - ATT_ASSERT_STATUS(mjb_grapheme_count(embedded_nul, 3, MJB_ENC_UTF_8, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count(embedded_nul, 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: embedded NUL status") ATT_ASSERT(count, (size_t)3, "Count: embedded NUL") // MJB_NUL_TERMINATED requests a terminator scan - ATT_ASSERT_STATUS(mjb_grapheme_count("hi", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, &count), + ATT_ASSERT_STATUS(mjb_grapheme_count("hi", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: NUL-terminated status") ATT_ASSERT(count, (size_t)2, "Count: NUL-terminated") // UTF-16LE input const char utf16le_ab[] = { 'A', '\0', 'B', '\0' }; - ATT_ASSERT_STATUS(mjb_grapheme_count(utf16le_ab, 4, MJB_ENC_UTF_16LE, &count), MJB_STATUS_OK, + ATT_ASSERT_STATUS(mjb_grapheme_count(utf16le_ab, 4, MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, &count, NULL), MJB_STATUS_OK, "Count: UTF-16LE status") ATT_ASSERT(count, (size_t)2, "Count: UTF-16LE AB") } diff --git a/tests/string.c b/tests/string.c index 8d075768..def15b9a 100644 --- a/tests/string.c +++ b/tests/string.c @@ -38,7 +38,8 @@ static mjb_status test_output_failure_writer(mjb_output *output, const void *con static size_t count_codepoints(const char *buffer, size_t byte_length, mjb_encoding encoding) { size_t count = 0; - if(mjb_codepoint_count(buffer, byte_length, encoding, &count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(buffer, byte_length, encoding, MJB_MALFORMED_STOP, &count, NULL) != + MJB_STATUS_OK) { return SIZE_MAX; } @@ -110,7 +111,8 @@ int test_string(void *arg) { ATT_ASSERT(count_codepoints("Héllö", 7, enc), 5, "UTF-8 length: Héllö") ATT_ASSERT(count_codepoints("Héllö", 4, enc), 3, "UTF-8 length: Héllö") - ATT_ASSERT(count_codepoints("Héllö", 2, enc), 1, "UTF-8 length: Héllö") + ATT_ASSERT(count_codepoints("Héllö", 2, enc), SIZE_MAX, + "UTF-8 length rejects a truncated sequence") ATT_ASSERT(count_codepoints("Héllö", 0, enc), 0, "UTF-8 length: Héllö") ATT_ASSERT(count_codepoints("Hèllõ ツ", 11, enc), 7, "UTF-8 length: Hèllõ ツ") ATT_ASSERT(count_codepoints("Hèllõ ツ", 5, enc), 4, "UTF-8 length: Hèllõ ツ") @@ -120,14 +122,16 @@ int test_string(void *arg) { ATT_ASSERT(count_codepoints("Hello", 5, MJB_ENC_ASCII), 5, "ASCII length: Hello") size_t codepoint_count = 6251; - ATT_ASSERT_STATUS(mjb_codepoint_count("A", 1, MJB_ENC_UTF_8, NULL), + ATT_ASSERT_STATUS(mjb_codepoint_count("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "Codepoint count rejects NULL count") - ATT_ASSERT_STATUS(mjb_codepoint_count(NULL, 1, MJB_ENC_UTF_8, &codepoint_count), + ATT_ASSERT_STATUS(mjb_codepoint_count(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &codepoint_count, NULL), MJB_STATUS_INVALID_ARGUMENT, "Codepoint count rejects NULL buffer") ATT_ASSERT(codepoint_count, (size_t)0, "Codepoint count is zero after NULL buffer") codepoint_count = 6251; - ATT_ASSERT_STATUS(mjb_codepoint_count("Hello", 5, MJB_ENC_UNKNOWN, &codepoint_count), + ATT_ASSERT_STATUS(mjb_codepoint_count("Hello", 5, MJB_ENC_UNKNOWN, MJB_MALFORMED_STOP, + &codepoint_count, NULL), MJB_STATUS_INVALID_ENCODING, "Codepoint count rejects an unknown encoding") ATT_ASSERT(codepoint_count, (size_t)0, "Codepoint count is zero after invalid encoding") diff --git a/tests/terminal-width.c b/tests/terminal-width.c index 6de11be5..fa403e44 100644 --- a/tests/terminal-width.c +++ b/tests/terminal-width.c @@ -11,153 +11,171 @@ int test_terminal_width(void *arg) { size_t sw = 0; - ATT_ASSERT_STATUS(mjb_terminal_width("A", 1, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, NULL), + ATT_ASSERT_STATUS(mjb_terminal_width("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, NULL, NULL), MJB_STATUS_INVALID_ARGUMENT, "NULL width pointer") - ATT_ASSERT_STATUS(mjb_terminal_width(NULL, 1, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width(NULL, 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_INVALID_ARGUMENT, "NULL terminal buffer") - ATT_ASSERT_STATUS(mjb_terminal_width("A", 1, MJB_ENC_UTF_8, - (mjb_terminal_width_profile)2, &sw), MJB_STATUS_INVALID_ARGUMENT, + ATT_ASSERT_STATUS(mjb_terminal_width("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + (mjb_terminal_width_profile)2, &sw, NULL), MJB_STATUS_INVALID_ARGUMENT, "Invalid terminal-width profile") - ATT_ASSERT_STATUS(mjb_terminal_width("A", 1, MJB_ENC_UNKNOWN, MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("A", 1, MJB_ENC_UNKNOWN, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_INVALID_ENCODING, "Invalid input encoding") - ATT_ASSERT_STATUS(mjb_terminal_width("", 0, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("", 0, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Empty string") ATT_ASSERT(sw, 0, "Empty string") - ATT_ASSERT_STATUS(mjb_terminal_width(" ", 1, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width(" ", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Space") ATT_ASSERT(sw, 1, "Space") // Test combining marks (should be zero width) - ATT_ASSERT_STATUS(mjb_terminal_width("e\xCC\x81", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("e\xCC\x81", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "e + combining acute") ATT_ASSERT(sw, 1, "e + combining acute (é)") - ATT_ASSERT_STATUS(mjb_terminal_width("\xCC\x81", 2, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("\xCC\x81", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Combining acute alone") ATT_ASSERT(sw, 0, "Combining acute alone") // Test zero-width format characters - ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\x8B", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Zero-width space") + ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\x8B", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Zero-width space") ATT_ASSERT(sw, 0, "Zero-width space (U+200B)") - ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\x8C", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Zero-width non-joiner") + ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\x8C", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Zero-width non-joiner") ATT_ASSERT(sw, 0, "Zero-width non-joiner (U+200C)") - ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\x8D", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Zero-width joiner") + ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\x8D", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Zero-width joiner") ATT_ASSERT(sw, 0, "Zero-width joiner (U+200D)") - ATT_ASSERT_STATUS(mjb_terminal_width("\t", 1, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("\t", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_UNSUPPORTED, "Tab depends on terminal state") - ATT_ASSERT_STATUS(mjb_terminal_width("\n", 1, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("\n", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_UNSUPPORTED, "Newline is not single-line text") - ATT_ASSERT_STATUS(mjb_terminal_width("\x1B[31m", 5, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_UNSUPPORTED, + ATT_ASSERT_STATUS(mjb_terminal_width("\x1B[31m", 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_UNSUPPORTED, "ANSI escape sequence is not printable text") - ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\xA8", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_UNSUPPORTED, + ATT_ASSERT_STATUS(mjb_terminal_width("\xE2\x80\xA8", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_UNSUPPORTED, "Unicode line separator is not single-line text") ATT_ASSERT(sw, 0, "Failed measurement clears the output width") // Test wide characters (CJK) - ATT_ASSERT_STATUS(mjb_terminal_width("\xe4\xb8\xad", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "CJK ideograph") + ATT_ASSERT_STATUS(mjb_terminal_width("\xe4\xb8\xad", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "CJK ideograph") ATT_ASSERT(sw, 2, "CJK ideograph (中)") - ATT_ASSERT_STATUS(mjb_terminal_width("\xe4\xb8\xad\xe6\x96\x87", 6, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Two CJK ideographs") + ATT_ASSERT_STATUS(mjb_terminal_width("\xe4\xb8\xad\xe6\x96\x87", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Two CJK ideographs") ATT_ASSERT(sw, 4, "Two CJK ideographs (中文)") // Test full-width characters - ATT_ASSERT_STATUS(mjb_terminal_width("\xef\xbc\xa1", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Full-width A") + ATT_ASSERT_STATUS(mjb_terminal_width("\xef\xbc\xa1", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Full-width A") ATT_ASSERT(sw, 2, "Full-width A (Ａ)") // Test mixed strings - ATT_ASSERT_STATUS(mjb_terminal_width("Hello", 5, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("Hello", 5, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "ASCII string") ATT_ASSERT(sw, 5, "ASCII string (Hello)") - ATT_ASSERT_STATUS(mjb_terminal_width("Hello", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "NUL-terminated UTF-8") + ATT_ASSERT_STATUS(mjb_terminal_width("Hello", MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "NUL-terminated UTF-8") ATT_ASSERT(sw, 5, "NUL-terminated UTF-8 string") const char utf16le[] = { 0x41, 0x00, 0x4C, 0x75 }; - ATT_ASSERT_STATUS(mjb_terminal_width(utf16le, sizeof(utf16le), MJB_ENC_UTF_16LE, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "UTF-16LE input") + ATT_ASSERT_STATUS(mjb_terminal_width(utf16le, sizeof(utf16le), MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "UTF-16LE input") ATT_ASSERT(sw, 3, "UTF-16LE A + CJK") const char utf32be[] = { 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x75, 0x4C }; - ATT_ASSERT_STATUS(mjb_terminal_width(utf32be, sizeof(utf32be), MJB_ENC_UTF_32BE, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "UTF-32BE input") + ATT_ASSERT_STATUS(mjb_terminal_width(utf32be, sizeof(utf32be), MJB_ENC_UTF_32BE, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "UTF-32BE input") ATT_ASSERT(sw, 3, "UTF-32BE A + CJK") - ATT_ASSERT_STATUS(mjb_terminal_width("Hello\xe4\xb8\xad", 8, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), + ATT_ASSERT_STATUS(mjb_terminal_width("Hello\xe4\xb8\xad", 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "ASCII + CJK") ATT_ASSERT(sw, 7, "ASCII + CJK (Hello中)") - ATT_ASSERT_STATUS(mjb_terminal_width("e\xcc\x81\xe4\xb8\xad", 6, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Combining + CJK") + ATT_ASSERT_STATUS(mjb_terminal_width("e\xcc\x81\xe4\xb8\xad", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Combining + CJK") ATT_ASSERT(sw, 3, "Combining + CJK (é中)") // Test complex emojis // Simple emoji: 👍 (U+1F44D THUMBS UP SIGN) - EAW=Wide - ATT_ASSERT_STATUS(mjb_terminal_width("\xf0\x9f\x91\x8d", 4, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Simple emoji") + ATT_ASSERT_STATUS(mjb_terminal_width("\xf0\x9f\x91\x8d", 4, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Simple emoji") ATT_ASSERT(sw, 2, "Simple emoji (👍)") // Emoji with variation selector: ❤️ (U+2764 HEAVY BLACK HEART + U+FE0F VARIATION SELECTOR-16) - ATT_ASSERT_STATUS(mjb_terminal_width("\xe2\x9d\xa4\xef\xb8\x8f", 6, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Emoji with variation selector") + ATT_ASSERT_STATUS(mjb_terminal_width("\xe2\x9d\xa4\xef\xb8\x8f", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Emoji with variation selector") ATT_ASSERT(sw, 2, "Emoji presentation sequence (❤️)") - ATT_ASSERT_STATUS(mjb_terminal_width("\xe2\x9d\xa4\xef\xb8\x8e", 6, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Text variation selector") + ATT_ASSERT_STATUS(mjb_terminal_width("\xe2\x9d\xa4\xef\xb8\x8e", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Text variation selector") ATT_ASSERT(sw, 1, "Text presentation sequence (❤︎) retains base East Asian Width") // Emoji with skin tone modifier: 👋🏽 (U+1F44B WAVING HAND + U+1F3FD EMOJI MODIFIER FITZPATRICK // TYPE-4) - ATT_ASSERT_STATUS(mjb_terminal_width("\xf0\x9f\x91\x8b\xf0\x9f\x8f\xbd", 8, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Emoji with skin tone") + ATT_ASSERT_STATUS(mjb_terminal_width("\xf0\x9f\x91\x8b\xf0\x9f\x8f\xbd", 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Emoji with skin tone") ATT_ASSERT(sw, 2, "Emoji with skin tone (👋🏽) is one terminal glyph") // ZWJ sequence: 👨‍👩‍👧 (U+1F468 MAN + U+200D ZWJ + U+1F469 WOMAN + U+200D ZWJ + // U+1F467 GIRL) ATT_ASSERT_STATUS(mjb_terminal_width("\xf0\x9f\x91\xa8\xe2\x80\x8d\xf0\x9f\x91\xa9\xe2\x80\x8d" - "\xf0\x9f\x91\xa7", 18, MJB_ENC_UTF_8, MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, + "\xf0\x9f\x91\xa7", 18, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Family emoji ZWJ sequence") ATT_ASSERT(sw, 2, "Family emoji ZWJ sequence (👨‍👩‍👧)") const char *kiss = "\xf0\x9f\x91\xa8\xf0\x9f\x8f\xbb\xe2\x80\x8d\xe2\x9d\xa4\xef\xb8\x8f" "\xe2\x80\x8d\xf0\x9f\x92\x8b\xe2\x80\x8d\xf0\x9f\x91\xa8\xf0\x9f\x8f\xbb"; - ATT_ASSERT_STATUS(mjb_terminal_width(kiss, strlen(kiss), MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Ten-scalar kiss emoji") + ATT_ASSERT_STATUS(mjb_terminal_width(kiss, strlen(kiss), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Ten-scalar kiss emoji") ATT_ASSERT(sw, 2, "Ten-scalar kiss emoji occupies two terminal cells") - ATT_ASSERT_STATUS(mjb_terminal_width("\xc2\xa1", 2, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "Ambiguous (narrow profile)") + ATT_ASSERT_STATUS(mjb_terminal_width("\xc2\xa1", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "Ambiguous (narrow profile)") ATT_ASSERT(sw, 1, "Ambiguous character is narrow") - ATT_ASSERT_STATUS(mjb_terminal_width("\xc2\xa1", 2, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_EAST_ASIAN, &sw), MJB_STATUS_OK, "Ambiguous (East Asian profile)") + ATT_ASSERT_STATUS(mjb_terminal_width("\xc2\xa1", 2, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_EAST_ASIAN, &sw, NULL), MJB_STATUS_OK, "Ambiguous (East Asian profile)") ATT_ASSERT(sw, 2, "Ambiguous character is wide") - ATT_ASSERT_STATUS(mjb_terminal_width("\xEA\xB0\x80", 3, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "NFC Hangul syllable") + ATT_ASSERT_STATUS(mjb_terminal_width("\xEA\xB0\x80", 3, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "NFC Hangul syllable") ATT_ASSERT(sw, 2, "NFC Hangul syllable") - ATT_ASSERT_STATUS(mjb_terminal_width("\xE1\x84\x80\xE1\x85\xA1", 6, MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_OK, "NFD Hangul syllable") + ATT_ASSERT_STATUS(mjb_terminal_width("\xE1\x84\x80\xE1\x85\xA1", 6, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &sw, NULL), MJB_STATUS_OK, "NFD Hangul syllable") ATT_ASSERT(sw, 2, "Canonically equivalent Hangul has equal terminal width") const unsigned char malformed[] = { 0xC3, 0x28 }; + mjb_diagnostic diagnostic; ATT_ASSERT_STATUS(mjb_terminal_width((const char *)malformed, sizeof(malformed), MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &sw), MJB_STATUS_MALFORMED_INPUT, "Malformed UTF-8") + MJB_MALFORMED_STOP, MJB_TERMINAL_WIDTH_NARROW, &sw, &diagnostic), + MJB_STATUS_MALFORMED_INPUT, "Malformed UTF-8") + ATT_ASSERT(diagnostic.byte_offset, (size_t)0, "Terminal width malformed offset") + + const char recoverable_width[] = { 'A', (char)0x80, (char)0xE4, (char)0xB8, (char)0xAD }; + ATT_ASSERT_STATUS(mjb_terminal_width(recoverable_width, sizeof(recoverable_width), + MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE, MJB_TERMINAL_WIDTH_NARROW, &sw, + &diagnostic), + MJB_STATUS_OK, "Terminal width replaces malformed input") + ATT_ASSERT(sw, (size_t)4, "Terminal width counts replacement glyph") + ATT_ASSERT(diagnostic.byte_offset, (size_t)1, + "Terminal width replacement retains malformed offset") + + ATT_ASSERT_STATUS(mjb_terminal_width(recoverable_width, sizeof(recoverable_width), + MJB_ENC_UTF_8, MJB_MALFORMED_SKIP, MJB_TERMINAL_WIDTH_NARROW, &sw, + &diagnostic), + MJB_STATUS_OK, "Terminal width skips malformed input") + ATT_ASSERT(sw, (size_t)3, "Terminal width omits skipped malformed input") return 0; } diff --git a/tests/utf.c b/tests/utf.c index f4771aca..f91f9e22 100644 --- a/tests/utf.c +++ b/tests/utf.c @@ -11,7 +11,8 @@ static size_t count_codepoints(const char *buffer, size_t byte_length, mjb_encoding encoding) { size_t count = 0; - if(mjb_codepoint_count(buffer, byte_length, encoding, &count) != MJB_STATUS_OK) { + if(mjb_codepoint_count(buffer, byte_length, encoding, MJB_MALFORMED_STOP, &count, NULL) != + MJB_STATUS_OK) { return SIZE_MAX; } @@ -223,17 +224,17 @@ int test_utf(void *arg) { ATT_ASSERT(index, 4, "UTF-16LE surrogate: index 4") ATT_ASSERT(in_error, false, "UTF-16LE surrogate: not error state") - // Truncated trailing units must terminate decoding (one replacement, then end), not loop. + // Strict codepoint counting rejects truncated trailing code units. MJB_TEST_COVERAGE(mjb_codepoint_count); - ATT_ASSERT(count_codepoints("A\0B", 3, MJB_ENC_UTF_16BE), 2, - "UTF-16BE: truncated trailing unit ends decoding") - ATT_ASSERT(count_codepoints("A", 1, MJB_ENC_UTF_16LE), 1, - "UTF-16LE: lone trailing byte decodes as replacement") - ATT_ASSERT(count_codepoints("\0\0\0A!", 5, MJB_ENC_UTF_32BE), 2, - "UTF-32BE: truncated trailing unit ends decoding") - ATT_ASSERT(count_codepoints("A!\0", 3, MJB_ENC_UTF_32LE), 1, - "UTF-32LE: lone truncated unit decodes as replacement") + ATT_ASSERT(count_codepoints("A\0B", 3, MJB_ENC_UTF_16BE), SIZE_MAX, + "UTF-16BE: truncated trailing unit is malformed") + ATT_ASSERT(count_codepoints("A", 1, MJB_ENC_UTF_16LE), SIZE_MAX, + "UTF-16LE: lone trailing byte is malformed") + ATT_ASSERT(count_codepoints("\0\0\0A!", 5, MJB_ENC_UTF_32BE), SIZE_MAX, + "UTF-32BE: truncated trailing unit is malformed") + ATT_ASSERT(count_codepoints("A!\0", 3, MJB_ENC_UTF_32LE), SIZE_MAX, + "UTF-32LE: lone truncated unit is malformed") #undef RESET_STATE diff --git a/tests/utils/utils.c b/tests/utils/utils.c index 37930bc9..06516444 100644 --- a/tests/utils/utils.c +++ b/tests/utils/utils.c @@ -49,15 +49,16 @@ char *run_mjb_map_case(const char *buffer, size_t byte_length, mjb_map_case_type mjb_encoding encoding) { mjb_result result = { NULL, 0, false }; - if(mjb_map_case(buffer, byte_length, encoding, type, encoding, &result) != MJB_STATUS_OK) { + if(mjb_map_case(buffer, byte_length, encoding, MJB_MALFORMED_STOP, type, encoding, &result, + NULL) != MJB_STATUS_OK) { return NULL; } MJB_TEST_COVERAGE(mjb_map_case_into); size_t required = 0; - mjb_status status = mjb_map_case_into(buffer, byte_length, encoding, type, encoding, NULL, - &required); + mjb_status status = mjb_map_case_into(buffer, byte_length, encoding, MJB_MALFORMED_STOP, type, + encoding, NULL, &required, NULL); ATT_ASSERT_STATUS(status, MJB_STATUS_OK, "Case into sizing matches allocating case mapping") ATT_ASSERT(required, result.output_size, "Case into required size matches allocated size") @@ -75,7 +76,8 @@ char *run_mjb_map_case(const char *buffer, size_t byte_length, mjb_map_case_type memset(output, 0xA5, required + 1); size_t output_size = required; - status = mjb_map_case_into(buffer, byte_length, encoding, type, encoding, output, &output_size); + status = mjb_map_case_into(buffer, byte_length, encoding, MJB_MALFORMED_STOP, type, encoding, + output, &output_size, NULL); ATT_ASSERT_STATUS(status, MJB_STATUS_OK, "Case into output matches allocating case mapping") ATT_ASSERT(output_size, result.output_size, "Case into written size matches allocated size") diff --git a/utils/generate/functions.ts b/utils/generate/functions.ts index 9b98f77a..32b3c5a5 100644 --- a/utils/generate/functions.ts +++ b/utils/generate/functions.ts @@ -114,6 +114,25 @@ function encoding(description = 'The encoding of the string', name = 'encoding') } } +function malformedPolicy(description = 'How malformed code-unit sequences are handled'): MojibakeArg { + return { + name: 'malformed_policy', + type: 'mjb_malformed_policy', + description, + wasm_generated: false, + is_enum: true + }; +} + +function diagnostic(description = 'Where to store the first malformed-input diagnostic, or NULL'): MojibakeArg { + return { + name: 'diagnostic', + type: 'mjb_diagnostic *', + description, + wasm_generated: false + }; +} + function codepoint(description = 'The codepoint to check', name = 'codepoint'): MojibakeArg { return { name, @@ -214,6 +233,7 @@ printf("U+%04X lowercase: U+%04X", character.codepoint, character.lowercase);`, buffer('The string to normalize'), byte_length(), encoding(), + malformedPolicy(), { name: 'form', type: 'mjb_normalization', @@ -222,25 +242,32 @@ printf("U+%04X lowercase: U+%04X", character.codepoint, character.lowercase);`, is_enum: true }, encoding('The output encoding of the string', 'output_encoding'), - result() + result(), + diagnostic() ], wasm: true, section: Section.TextTransformation, details: 'Normalize a string to the requested Unicode normalization form. If the input is ' + 'already normalized and no encoding conversion is needed, the input buffer is returned ' + - 'as-is in `result->output` with `result->transformed` set to false, without allocating.', + 'as-is in `result->output` with `result->transformed` set to false, without allocating. ' + + 'Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one.', returns: [ { value: 'MJB_STATUS_OK', description: 'The string was normalized (or already normal)' }, - { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`result` is NULL, or `buffer` is NULL with a non-zero size' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`result` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'An encoding is invalid or lacks byte-order information' }, { value: 'MJB_STATUS_INVALID_FORM', description: '`form` is not NFC, NFD, NFKC, or NFKD' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The output size would overflow' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' } ], example: `const char *input = "Cafe\\xCC\\x81"; // "Cafe" + U+0301 COMBINING ACUTE ACCENT mjb_result result; -if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_normalize(input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -261,6 +288,7 @@ mjb_result_free(&result);`, buffer('The string to normalize'), byte_length(), encoding(), + malformedPolicy(), { name: 'form', type: 'mjb_normalization', @@ -281,7 +309,8 @@ mjb_result_free(&result);`, type: 'size_t *', description: 'The input capacity and output required or written byte count', wasm_generated: false - } + }, + diagnostic() ], wasm: false, section: Section.TextTransformation, @@ -296,13 +325,13 @@ mjb_result_free(&result);`, { value: 'MJB_STATUS_OK', description: 'The required size was returned or the normalized string was written' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`output_size` is NULL, or `buffer` is NULL with a non-zero size' }, + '`output_size` is NULL, the buffer is invalid, or the malformed policy is invalid' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: 'An encoding is invalid or lacks byte-order information' }, { value: 'MJB_STATUS_INVALID_FORM', description: '`form` is not NFC, NFD, NFKC, or NFKD' }, { value: 'MJB_STATUS_MALFORMED_INPUT', description: - 'The input contains an ill-formed code-unit sequence' }, + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The requested output encoding cannot represent a normalized codepoint' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The required output size would overflow' }, @@ -313,15 +342,16 @@ mjb_result_free(&result);`, example: `const char *input = "Cafe\\xCC\\x81"; // "Cafe" + U+0301 COMBINING ACUTE ACCENT size_t output_size = 0; -if(mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, - MJB_ENC_UTF_8, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } char output[5]; if(output_size > sizeof(output) || mjb_normalize_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, output, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } @@ -340,6 +370,7 @@ printf("NFC payload (no terminator): %.*s", (int)output_size, output);`, buffer('The string to filter'), byte_length(), encoding(), + malformedPolicy(), { name: 'filters', type: 'mjb_filter_flags', @@ -347,18 +378,35 @@ printf("NFC payload (no terminator): %.*s", (int)output_size, output);`, wasm_generated: false }, encoding('The output encoding of the string', 'output_encoding'), - result() + result(), + diagnostic() ], details: '`MJB_FILTER_LIMIT_COMBINING` removes combining marks after the first ' + '`MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in an emitted run. This is useful ' + - 'for reducing Zalgo-style text while keeping ordinary accents and stacked marks.', + 'for reducing Zalgo-style text while keeping ordinary accents and stacked marks. ' + + 'Malformed subsequences are stopped, replaced, or skipped according to ' + + '`malformed_policy`; `diagnostic` records the first one encountered.', + returns: [ + { value: 'MJB_STATUS_OK', description: 'The filtered string was returned' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`result` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, + { value: 'MJB_STATUS_UNSUPPORTED', description: + 'The requested output encoding cannot represent a filtered codepoint' }, + { value: 'MJB_STATUS_OVERFLOW', description: 'The output size would overflow' }, + { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' } + ], wasm: true, section: Section.TextTransformation, example: `const char *mixed_whitespace = "Hello\\t\\t\\n\\nworld"; mjb_result result; if(mjb_filter(mixed_whitespace, strlen(mixed_whitespace), MJB_ENC_UTF_8, - MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, &result) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, &result, + NULL) != MJB_STATUS_OK) { return 1; } @@ -369,8 +417,8 @@ mjb_result_free(&result); const char *controls = "\\x1\\x2\\t\\n\\v\\f\\r\\x1f"; -if(mjb_filter(controls, strlen(controls), MJB_ENC_UTF_8, MJB_FILTER_CONTROLS, - MJB_ENC_UTF_8, &result) != MJB_STATUS_OK) { +if(mjb_filter(controls, strlen(controls), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_CONTROLS, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -389,6 +437,7 @@ mjb_result_free(&result);`, buffer('The string to filter'), byte_length(), encoding(), + malformedPolicy(), { name: 'filters', type: 'mjb_filter_flags', @@ -408,7 +457,8 @@ mjb_result_free(&result);`, type: 'size_t *', description: 'The input capacity and output required or written byte count', wasm_generated: false - } + }, + diagnostic() ], wasm: false, section: Section.TextTransformation, @@ -419,14 +469,17 @@ mjb_result_free(&result);`, 'are not written. No bytes are written when capacity is insufficient. Filtering itself ' + 'does not allocate, but `MJB_FILTER_NORMALIZE` may allocate temporary normalization ' + 'storage. `MJB_FILTER_LIMIT_COMBINING` keeps the first ' + - '`MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in each emitted run.', + '`MJB_FILTER_MAX_COMBINING_MARKS` consecutive marks in each emitted run. Malformed ' + + 'subsequences follow `malformed_policy`, and `diagnostic` records the first one.', returns: [ { value: 'MJB_STATUS_OK', description: 'The required size was returned or the filtered string was written' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`output_size` is NULL, or `buffer` is NULL with a non-zero size' }, + '`output_size` is NULL, the buffer is invalid, or the malformed policy is invalid' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The requested output encoding cannot represent a filtered codepoint' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The required output size would overflow' }, @@ -438,15 +491,17 @@ mjb_result_free(&result);`, example: `const char *input = "Hello\\t\\t\\nworld"; size_t output_size = 0; -if(mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, MJB_FILTER_COLLAPSE_SPACES, - MJB_ENC_UTF_8, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, NULL, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } char output[11]; if(output_size > sizeof(output) || mjb_filter_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_FILTER_COLLAPSE_SPACES, MJB_ENC_UTF_8, output, + &output_size, NULL) != MJB_STATUS_OK) { return 1; } @@ -463,18 +518,26 @@ printf("Filtered payload (no terminator): %.*s", (int)output_size, output);`, buffer('The string to transform'), byte_length(), encoding(), + malformedPolicy(), encoding('The output encoding of the string', 'output_encoding'), - result() + result(), + diagnostic() ], wasm: true, section: Section.TextTransformation, details: 'Apply the normative `NFKC_Casefold` mapping and normalize the result to NFC. ' + 'This transform performs compatibility folding, full default case folding, and removal ' + 'of default-ignorable codepoints. It is intended for identifier comparison and is not ' + - 'locale-sensitive.', + 'locale-sensitive. Malformed subsequences follow `malformed_policy`, and `diagnostic` ' + + 'records the first one.', returns: [ { value: 'MJB_STATUS_OK', description: 'The transformed string was returned' }, - { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`result` is NULL, or `buffer` is NULL with a non-zero size' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`result` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The output size would overflow' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The transform did not stabilize' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' } @@ -482,8 +545,8 @@ printf("Filtered payload (no terminator): %.*s", (int)output_size, output);`, example: `const char *input = "Stra\\xC3\\x9F" "e\\xC2\\xAD"; mjb_result result; -if(mjb_nfkc_casefold(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_nfkc_casefold(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -506,6 +569,7 @@ mjb_result_free(&result);`, buffer('The string to transform'), byte_length(), encoding(), + malformedPolicy(), encoding('The output encoding of the string', 'output_encoding'), { name: 'output', @@ -519,7 +583,8 @@ mjb_result_free(&result);`, type: 'size_t *', description: 'The input capacity and output required or written byte count', wasm_generated: false - } + }, + diagnostic() ], wasm: false, section: Section.TextTransformation, @@ -534,7 +599,11 @@ mjb_result_free(&result);`, { value: 'MJB_STATUS_OK', description: 'The required size was returned or the transformed string was written' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`output_size` is NULL, or `buffer` is NULL with a non-zero size' }, + '`output_size` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The required output size would overflow' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The transform did not stabilize' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Temporary allocation failed' }, @@ -544,15 +613,15 @@ mjb_result_free(&result);`, example: `const char *input = "Stra\\xC3\\x9F" "e\\xC2\\xAD"; size_t output_size = 0; -if(mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, - NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } char output[7]; if(output_size > sizeof(output) || mjb_nfkc_casefold_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_ENC_UTF_8, output, &output_size, NULL) != MJB_STATUS_OK) { return 1; } @@ -923,6 +992,120 @@ printf("Valid UTF-8: %s", mjb_is_utf8(input, strlen(input)) ? "yes" : "no");` // UTF-16: yes printf("UTF-16: %s", mjb_is_utf16(utf16be, sizeof(utf16be) - 1) ? "yes" : "no");` }, + { + comment: 'Validate a complete Unicode code-unit sequence.', + ret: 'mjb_status', + name: 'mjb_string_validate', + attributes: ['MJB_NODISCARD'], + args: [ + buffer('The string to validate'), + byte_length(), + encoding(), + diagnostic() + ], + wasm: false, + section: Section.TextAnalysis, + details: 'Validate the complete input without producing output. Empty input is well-formed. ' + + 'On malformed input, `diagnostic` identifies the first maximal ill-formed subsequence. ' + + 'Generic UTF-16 and UTF-32 require a byte-order mark.', + returns: [ + { value: 'MJB_STATUS_OK', description: 'The complete input is well-formed' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`buffer` is NULL with a non-zero size' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'The encoding is unsupported, or generic UTF-16/UTF-32 has no byte-order mark' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'The first malformed subsequence is described by `diagnostic`' } + ], + example: `const char invalid[] = "\\xE2\\x82"; +mjb_diagnostic diagnostic; + +if(mjb_string_validate(invalid, sizeof(invalid) - 1, MJB_ENC_UTF_8, + &diagnostic) != MJB_STATUS_MALFORMED_INPUT || diagnostic.byte_offset != 0) { + return 1; +}`, + related: ['mjb_decode_next', 'mjb_decode_previous', 'mjb_is_utf8'] + }, + { + comment: 'Decode the next codepoint from a string.', + ret: 'mjb_status', + name: 'mjb_decode_next', + attributes: ['MJB_NODISCARD'], + args: [ + buffer('The string to decode'), + byte_length(), + encoding(), + malformedPolicy(), + { + name: 'offset', + type: 'size_t *', + description: 'The input byte offset and the byte offset following the decoded subsequence', + wasm_generated: false + }, + { + name: 'codepoint', + type: 'mjb_codepoint *', + description: 'Where to store the decoded codepoint', + wasm_generated: false + }, + diagnostic() + ], + wasm: false, + section: Section.TextAnalysis, + details: 'Decode one codepoint and advance `offset`. With `MJB_MALFORMED_STOP`, malformed ' + + 'input returns `MJB_STATUS_MALFORMED_INPUT`; `offset` still advances over the malformed ' + + 'subsequence so decoding can resume. `MJB_MALFORMED_REPLACE` returns U+FFFD, while ' + + '`MJB_MALFORMED_SKIP` advances until a valid codepoint or end of input. A diagnostic is ' + + 'reported for replacement and skipping even when the function returns success.', + returns: [ + { value: 'MJB_STATUS_OK', description: 'A codepoint was decoded' }, + { value: 'MJB_STATUS_END_OF_INPUT', description: 'No codepoint remains' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: 'An argument or policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding cannot be decoded' } + ], + related: ['mjb_decode_previous', 'mjb_string_validate', 'mjb_codepoint_count'] + }, + { + comment: 'Decode the previous codepoint from a string.', + ret: 'mjb_status', + name: 'mjb_decode_previous', + attributes: ['MJB_NODISCARD'], + args: [ + buffer('The string to decode'), + byte_length(), + encoding(), + malformedPolicy(), + { + name: 'offset', + type: 'size_t *', + description: 'The input byte offset and the start of the decoded subsequence', + wasm_generated: false + }, + { + name: 'codepoint', + type: 'mjb_codepoint *', + description: 'Where to store the decoded codepoint', + wasm_generated: false + }, + diagnostic() + ], + wasm: false, + section: Section.TextAnalysis, + details: 'Decode backward from `offset`, using the same malformed-input policies and ' + + 'diagnostic contract as `mjb_decode_next`. On success, `offset` is the first byte of the ' + + 'decoded codepoint. Start with the input byte length to iterate from the end.', + returns: [ + { value: 'MJB_STATUS_OK', description: 'A codepoint was decoded' }, + { value: 'MJB_STATUS_END_OF_INPUT', description: 'No codepoint precedes `offset`' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: 'An argument or policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding cannot be decoded' } + ], + related: ['mjb_decode_next', 'mjb_string_validate', 'mjb_codepoint_count'] + }, { comment: 'Count the codepoints in a string.', ret: 'mjb_status', @@ -932,22 +1115,26 @@ printf("UTF-16: %s", mjb_is_utf16(utf16be, sizeof(utf16be) - 1) ? "yes" : "no"); buffer('The string to count'), byte_length(), encoding(), + malformedPolicy(), { name: 'count', type: 'size_t *', description: 'The number of codepoints to store; set to zero on failure', wasm_generated: true - } + }, + diagnostic() ], wasm: true, section: Section.TextAnalysis, - details: 'Count the number of Unicode codepoints in a string. Malformed code-unit sequences ' + - 'count per the library replacement policy, and an incomplete trailing sequence does not ' + - 'add a codepoint. On failure, `count` is set to zero.', + details: 'Count the number of decoded Unicode codepoints in a string. Malformed subsequences ' + + 'are stopped, replaced, or skipped according to `malformed_policy`. A replacement counts ' + + 'as one codepoint. On failure, `count` is set to zero.', returns: [ { value: 'MJB_STATUS_OK', description: 'The count was computed' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`count` is NULL, or `buffer` is NULL with a non-zero size' }, - { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' } + { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' } ], example: `// The "Héllö" string is five Unicode characters, but has different byte lengths in different encodings. @@ -955,14 +1142,16 @@ const char *utf8 = "H\\xC3\\xA9ll\\xC3\\xB6"; // 7 bytes const char utf16le[] = "H\\0\\xE9\\0l\\0l\\0\\xF6\\0"; // 10 bytes size_t count; -if(mjb_codepoint_count(utf8, 7, MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_codepoint_count(utf8, 7, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } // 5 UTF-8 characters printf("%zu UTF-8 characters", count); -if(mjb_codepoint_count(utf16le, 10, MJB_ENC_UTF_16LE, &count) != MJB_STATUS_OK) { +if(mjb_codepoint_count(utf16le, 10, MJB_ENC_UTF_16LE, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -979,16 +1168,32 @@ printf("%zu UTF-16LE characters", count);`, buffer('The string to check'), byte_length(), encoding(), + malformedPolicy(), { name: 'callback', type: 'mjb_for_each_codepoint_fn', description: 'The function to call for each codepoint', wasm_generated: true - } + }, + diagnostic() ], wasm: true, section: Section.TextAnalysis, - example: `mjb_status status = mjb_for_each_codepoint("ABC", 3, MJB_ENC_UTF_8, NULL); + details: 'Decode the string according to `malformed_policy` and call the callback for every ' + + 'resulting codepoint. The first malformed subsequence is reported in `diagnostic` even ' + + 'when it is replaced or skipped.', + returns: [ + { value: 'MJB_STATUS_OK', description: 'Every decoded codepoint was visited' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + 'The buffer, callback, or malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'The encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, + { value: 'MJB_STATUS_CALLBACK_STOPPED', description: 'The callback returned false' } + ], + example: `mjb_status status = mjb_for_each_codepoint("ABC", 3, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, NULL, NULL); // A callback is required: yes bool callback_required = status == MJB_STATUS_INVALID_ARGUMENT; @@ -1219,8 +1424,10 @@ printf("%.*s sign uses %u UTF-8 bytes", (int)size, encoded, size);` buffer('The string to convert'), byte_length(), encoding('The input encoding of the string'), + malformedPolicy(), encoding('The output encoding of the string', 'output_encoding'), - result() + result(), + diagnostic() ], wasm: true, section: Section.TextTransformation, @@ -1228,13 +1435,16 @@ printf("%.*s sign uses %u UTF-8 bytes", (int)size, encoded, size);` 'UTF-32LE/BE). Generic UTF-16/UTF-32 input consumes a leading BOM as the encoding scheme ' + 'signature and uses it to resolve byte order. Explicit-endian input preserves an initial ' + 'U+FEFF as text. Generic UTF-16/UTF-32 without a BOM, and generic UTF-16/UTF-32 output, are ' + - 'rejected because the byte order is not specified.', + 'rejected because the byte order is not specified. Malformed source subsequences are ' + + 'stopped, replaced, or skipped according to `malformed_policy`.', returns: [ { value: 'MJB_STATUS_OK', description: 'The string was converted' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`result` is NULL, `buffer` is NULL with a non-zero size, or the input is not valid in the source encoding' }, + '`result` is NULL, `buffer` is NULL with a non-zero size, or the malformed policy is invalid' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: - 'A generic UTF-16/UTF-32 encoding did not provide enough byte order information' }, + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The requested encoding conversion is not supported' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The output size would overflow' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' } @@ -1242,8 +1452,8 @@ printf("%.*s sign uses %u UTF-8 bytes", (int)size, encoded, size);` example: `const char *input = "caf\\xC3\\xA9"; mjb_result result; -if(mjb_convert_encoding(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, &result) != MJB_STATUS_OK) { +if(mjb_convert_encoding(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -1261,6 +1471,7 @@ mjb_result_free(&result);`, buffer('The string to convert'), byte_length(), encoding('The input encoding of the string'), + malformedPolicy(), encoding('The output encoding of the string', 'output_encoding'), { name: 'output', @@ -1274,7 +1485,8 @@ mjb_result_free(&result);`, type: 'size_t *', description: 'The input capacity and output required or written byte count', wasm_generated: false - } + }, + diagnostic() ], wasm: false, section: Section.TextTransformation, @@ -1291,7 +1503,9 @@ mjb_result_free(&result);`, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`output_size` is NULL, or `buffer` is NULL with a non-zero size' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: - 'A generic UTF-16/UTF-32 encoding did not provide enough byte order information' }, + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The requested encoding conversion is not supported' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The required output size would overflow' }, @@ -1301,15 +1515,16 @@ mjb_result_free(&result);`, example: `const char *input = "caf\\xC3\\xA9"; size_t output_size = 0; -if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_ENC_UTF_16LE, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_convert_encoding_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } unsigned char output[8]; if(output_size > sizeof(output) || mjb_convert_encoding_into(input, strlen(input), - MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, output, &output_size) != MJB_STATUS_OK) { + MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, output, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } @@ -1466,6 +1681,7 @@ printf("apple sorts before banana: %s", order < 0 ? "yes" : "no");`, buffer('The string to generate the sort key for'), byte_length(), encoding(), + malformedPolicy(), { name: 'variable_weighting', type: 'mjb_collation_variable_weighting', @@ -1480,7 +1696,8 @@ printf("apple sorts before banana: %s", order < 0 ? "yes" : "no");`, wasm_generated: false, is_enum: true }, - result('The pointer to store the binary sort key') + result('The pointer to store the binary sort key'), + diagnostic() ], wasm: true, section: Section.SortingComparison, @@ -1489,18 +1706,20 @@ printf("apple sorts before banana: %s", order < 0 ? "yes" : "no");`, 'compared with `memcmp` and yield the same order as `mjb_collation_compare` when both use ' + 'the same variable weighting and strength. Useful when the same strings are compared many ' + 'times, such as sorting or database indexing. Empty input and non-empty input with no ' + - 'effective weights at the selected strength both produce a zero-length key. If ' + + 'effective weights at the selected strength both produce a zero-length key. Malformed ' + + 'subsequences follow `malformed_policy`, and `diagnostic` records the first one. If ' + '`MJB_FEATURE_COLLATION=0` the function always returns ' + '`MJB_STATUS_FEATURE_NOT_ENABLED`.', returns: [ { value: 'MJB_STATUS_OK', description: 'The sort key was generated' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`result` is NULL, `buffer` is NULL with a non-zero size, or an option is invalid' }, + '`result` is NULL, the buffer is invalid, the malformed policy is invalid, or a ' + + 'collation option is invalid' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The input encoding is invalid or lacks byte-order information' }, { value: 'MJB_STATUS_MALFORMED_INPUT', description: - 'The input contains an ill-formed code-unit sequence' }, + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The sort key size would overflow' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' }, { value: 'MJB_STATUS_FEATURE_NOT_ENABLED', description: @@ -1509,7 +1728,8 @@ printf("apple sorts before banana: %s", order < 0 ? "yes" : "no");`, example: `mjb_result key; if(mjb_collation_key("r\\xC3\\xA9sum\\xC3\\xA9", 8, MJB_ENC_UTF_8, - MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &key) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, &key, + NULL) != MJB_STATUS_OK) { return 1; } @@ -1528,6 +1748,7 @@ mjb_result_free(&key);`, buffer('The string to generate the sort key for'), byte_length(), encoding(), + malformedPolicy(), { name: 'variable_weighting', type: 'mjb_collation_variable_weighting', @@ -1554,7 +1775,8 @@ mjb_result_free(&key);`, type: 'size_t *', description: 'The input capacity and output required or written byte count', wasm_generated: false - } + }, + diagnostic() ], wasm: false, section: Section.SortingComparison, @@ -1572,10 +1794,11 @@ mjb_result_free(&key);`, { value: 'MJB_STATUS_OK', description: 'The required size was returned or the binary sort key was written' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`output_size` is NULL, `buffer` is NULL with a non-zero size, or an option is invalid' }, + '`output_size` is NULL, the buffer is invalid, the malformed policy is invalid, or a ' + + 'collation option is invalid' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The input encoding is invalid' }, { value: 'MJB_STATUS_MALFORMED_INPUT', description: - 'The input contains an ill-formed code-unit sequence' }, + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The required key size would overflow' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Temporary allocation failed' }, { value: 'MJB_STATUS_OUTPUT_TOO_SMALL', description: @@ -1586,16 +1809,18 @@ mjb_result_free(&key);`, example: `const char *input = "r\\xC3\\xA9sum\\xC3\\xA9"; size_t output_size = 0; -if(mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, MJB_COLLATION_NON_IGNORABLE, - MJB_COLLATION_TERTIARY, NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, NULL, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } unsigned char output[64]; if(output_size > sizeof(output) || mjb_collation_key_into(input, 8, MJB_ENC_UTF_8, + MJB_MALFORMED_STOP, MJB_COLLATION_NON_IGNORABLE, MJB_COLLATION_TERTIARY, output, - &output_size) != MJB_STATUS_OK) { + &output_size, NULL) != MJB_STATUS_OK) { return 1; } @@ -1613,6 +1838,7 @@ printf("Sort key is non-empty: %s", output_size > 0 ? "yes" : "no");`, buffer('The string to change case'), byte_length(), encoding(), + malformedPolicy(), { name: 'type', type: 'mjb_map_case_type', @@ -1621,7 +1847,8 @@ printf("Sort key is non-empty: %s", output_size > 0 ? "yes" : "no");`, is_enum: true }, encoding('The output encoding of the string', 'output_encoding'), - result() + result(), + diagnostic() ], wasm: true, section: Section.TextTransformation, @@ -1634,18 +1861,24 @@ printf("Sort key is non-empty: %s", output_size > 0 ? "yes" : "no");`, 'default non-Turkic mappings. `MJB_LOCALE_TR` and `MJB_LOCALE_AZ` apply ' + 'Turkish/Azerbaijani dotted-I casing and Turkic `T` case-folding mappings. ' + '`MJB_LOCALE_LT` applies Lithuanian dot-above casing rules, while case folding remains ' + - 'the default non-Turkic mapping.', + 'the default non-Turkic mapping. Malformed subsequences follow `malformed_policy`, and ' + + '`diagnostic` records the first one.', returns: [ { value: 'MJB_STATUS_OK', description: 'The case conversion succeeded' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`result` is NULL, `buffer` is NULL with a non-zero size, or `type` is not a valid case type' }, + '`result` is NULL, the buffer is invalid, the malformed policy is invalid, or `type` is ' + + 'not a valid case type' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' } ], example: `const char *input = "Stra\\xC3\\x9F""e"; // "Straße" mjb_result result; -if(mjb_map_case(input, strlen(input), MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - &result) != MJB_STATUS_OK) { +if(mjb_map_case(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_CASE_UPPER, + MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) { return 1; } @@ -1665,6 +1898,7 @@ mjb_result_free(&result);`, buffer('The string to change case'), byte_length(), encoding(), + malformedPolicy(), { name: 'type', type: 'mjb_map_case_type', @@ -1684,7 +1918,8 @@ mjb_result_free(&result);`, type: 'size_t *', description: 'The input capacity and output required or written byte count', wasm_generated: false - } + }, + diagnostic() ], wasm: false, section: Section.TextTransformation, @@ -1698,7 +1933,12 @@ mjb_result_free(&result);`, { value: 'MJB_STATUS_OK', description: 'The required size was returned or the case-mapped string was written' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: - '`output_size` is NULL, `buffer` is NULL with a non-zero size, or `type` is invalid' }, + '`output_size` is NULL, the buffer is invalid, the malformed policy is invalid, or ' + + '`type` is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'An encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The requested output encoding cannot represent a mapped codepoint' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The required output size would overflow' }, @@ -1708,15 +1948,16 @@ mjb_result_free(&result);`, example: `const char *input = "Stra\\xC3\\x9F""e"; // "Straße" size_t output_size = 0; -if(mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, MJB_CASE_UPPER, MJB_ENC_UTF_8, - NULL, &output_size) != MJB_STATUS_OK) { +if(mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_CASE_UPPER, MJB_ENC_UTF_8, NULL, &output_size, NULL) != MJB_STATUS_OK) { return 1; } char output[7]; if(output_size > sizeof(output) || mjb_map_case_into(input, strlen(input), MJB_ENC_UTF_8, - MJB_CASE_UPPER, MJB_ENC_UTF_8, output, &output_size) != MJB_STATUS_OK) { + MJB_MALFORMED_STOP, MJB_CASE_UPPER, MJB_ENC_UTF_8, output, &output_size, + NULL) != MJB_STATUS_OK) { return 1; } @@ -1979,28 +2220,34 @@ printf("Sentence-break positions: %zu", boundaries);`, buffer('The string to count'), byte_length(), encoding(), + malformedPolicy(), { name: 'count', type: 'size_t *', description: 'The number of sentence segments to store; set to zero on failure', wasm_generated: true - } + }, + diagnostic() ], wasm: true, section: Section.Segmentation, details: 'Count the sentence segments produced by the default Unicode sentence-boundary ' + 'rules. The default rules carry no abbreviation list, so text such as `Dr. Smith` counts ' + - 'as two sentences. Malformed code-unit sequences are segmented per the library ' + - 'replacement policy. On failure, `count` is set to zero.', + 'as two sentences. Malformed subsequences follow `malformed_policy`, and `diagnostic` ' + + 'records the first one. On failure, `count` is set to zero.', returns: [ { value: 'MJB_STATUS_OK', description: 'The count was computed' }, - { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`count` is NULL, or `buffer` is NULL with a non-zero size' }, - { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' } + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`count` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' } ], example: `const char *input = "Hello. How are you? Fine!"; size_t count; -if(mjb_sentence_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_sentence_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -2079,28 +2326,33 @@ printf("First two graphemes use %zu bytes", bytes);` buffer('The string to count'), byte_length(), encoding(), + malformedPolicy(), { name: 'count', type: 'size_t *', description: 'The number of grapheme clusters to store; set to zero on failure', wasm_generated: true - } + }, + diagnostic() ], wasm: true, section: Section.Segmentation, details: 'Count user-perceived characters: the number of extended grapheme cluster segments ' + - 'in the string. Malformed code-unit sequences are segmented per the library replacement ' + - 'policy, and an incomplete trailing sequence does not add a cluster. On failure, `count` ' + - 'is set to zero.', + 'in the string. Malformed subsequences follow `malformed_policy`, and `diagnostic` records ' + + 'the first one. On failure, `count` is set to zero.', returns: [ { value: 'MJB_STATUS_OK', description: 'The count was computed' }, - { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`count` is NULL, or `buffer` is NULL with a non-zero size' }, - { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' } + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`count` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' } ], example: `const char *input = "A\\xF0\\x9F\\x87\\xAE\\xF0\\x9F\\x87\\xB9"; // A🇮🇹 size_t count; -if(mjb_grapheme_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_grapheme_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -2177,12 +2429,14 @@ printf("First word segment uses %zu bytes", bytes);` buffer('The string to count'), byte_length(), encoding(), + malformedPolicy(), { name: 'count', type: 'size_t *', description: 'The number of word-like segments to store; set to zero on failure', wasm_generated: true - } + }, + diagnostic() ], wasm: true, section: Section.Segmentation, @@ -2194,17 +2448,21 @@ printf("First word segment uses %zu bytes", bytes);` 'scripts segmented by dictionary lookup in other implementations, such as Chinese, ' + 'Japanese, Thai, Lao, Khmer, and Burmese, the count approximates one word per character: ' + 'Mojibake does not use frequency dictionaries to keep the size of the library small. ' + - 'Malformed code-unit sequences are segmented per the library replacement policy. On ' + - 'failure, `count` is set to zero.', + 'Malformed subsequences follow `malformed_policy`, and `diagnostic` records the first one. ' + + 'On failure, `count` is set to zero.', returns: [ { value: 'MJB_STATUS_OK', description: 'The count was computed' }, - { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`count` is NULL, or `buffer` is NULL with a non-zero size' }, - { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' } + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`count` is NULL, the buffer is invalid, or the malformed policy is invalid' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is not a supported input encoding' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' } ], example: `const char *input = "Hello, world! It works."; size_t count; -if(mjb_word_count(input, strlen(input), MJB_ENC_UTF_8, &count) != MJB_STATUS_OK) { +if(mjb_word_count(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + &count, NULL) != MJB_STATUS_OK) { return 1; } @@ -2281,6 +2539,10 @@ printf("Six columns include %zu bytes", bytes);` { value: 'MJB_STATUS_OK', description: 'The paragraph was resolved' }, { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`result` is NULL, or `buffer` is NULL with a non-zero size' }, + { value: 'MJB_STATUS_INVALID_ENCODING', description: + 'The input encoding is invalid or lacks byte-order information' }, + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'The input contains an ill-formed code-unit sequence' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The paragraph size would overflow' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'Allocation failed' } ], @@ -3181,6 +3443,7 @@ printf("U+754C is wide: %s", width == MJB_EAW_WIDE ? "yes" : "no");`, buffer('The printable, single-line string to measure'), byte_length(), encoding(), + malformedPolicy(), { name: 'profile', type: 'mjb_terminal_width_profile', @@ -3193,7 +3456,8 @@ printf("U+754C is wide: %s", width == MJB_EAW_WIDE ? "yes" : "no");`, type: 'size_t *', description: 'The number of terminal cells to store; set to zero on failure', wasm_generated: true - } + }, + diagnostic() ], wasm: true, section: Section.TerminalWidth, @@ -3204,21 +3468,24 @@ printf("U+754C is wide: %s", width == MJB_EAW_WIDE ? "yes" : "no");`, 'This is a deterministic terminal policy, not a measurement of proportional glyph advances. ' + 'Use grapheme boundaries for cursor movement, selection, deletion, and user-perceived ' + 'character counts. Controls, line separators, and paragraph separators are rejected because ' + - 'their effect depends on terminal state. On failure, `width` is set to zero.', + 'their effect depends on terminal state. Malformed subsequences follow `malformed_policy`, ' + + 'and `diagnostic` records the first one. On failure, `width` is set to zero.', returns: [ { value: 'MJB_STATUS_OK', description: 'The width was computed' }, - { value: 'MJB_STATUS_INVALID_ARGUMENT', description: '`width` is NULL, the profile is invalid, or `buffer` is NULL with a non-zero size' }, + { value: 'MJB_STATUS_INVALID_ARGUMENT', description: + '`width` is NULL, the profile or malformed policy is invalid, or the buffer is invalid' }, { value: 'MJB_STATUS_INVALID_ENCODING', description: 'The encoding is invalid or lacks required byte-order information' }, { value: 'MJB_STATUS_UNSUPPORTED', description: 'The input contains a control, line separator, or paragraph separator' }, { value: 'MJB_STATUS_NO_MEMORY', description: 'NFC normalization could not allocate memory' }, { value: 'MJB_STATUS_OVERFLOW', description: 'The width would overflow' }, - { value: 'MJB_STATUS_MALFORMED_INPUT', description: 'The input contains a malformed code-unit sequence' } + { value: 'MJB_STATUS_MALFORMED_INPUT', description: + 'Malformed input was encountered with `MJB_MALFORMED_STOP`' } ], example: `const char *input = "A\\xE7\\x95\\x8C"; // A界 size_t width; -if(mjb_terminal_width(input, strlen(input), MJB_ENC_UTF_8, - MJB_TERMINAL_WIDTH_NARROW, &width) != MJB_STATUS_OK) { +if(mjb_terminal_width(input, strlen(input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_TERMINAL_WIDTH_NARROW, &width, NULL) != MJB_STATUS_OK) { return 1; } @@ -3377,8 +3644,9 @@ printf("Current locale reset to English: %s", locale == MJB_LOCALE_EN ? "yes" : ], example: `mjb_result result; -if(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_ENC_UTF_16LE, - &result) != MJB_STATUS_OK || mjb_result_free(&result) != MJB_STATUS_OK) { +if(mjb_convert_encoding("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, + MJB_ENC_UTF_16LE, &result, NULL) != MJB_STATUS_OK || + mjb_result_free(&result) != MJB_STATUS_OK) { return 1; } From 7a834a65217df8e12395d2d4242b9eca7de2c278 Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Sat, 22 Aug 2026 23:23:18 +0200 Subject: [PATCH 2/7] Fix: GCC unused return value --- src/utf.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utf.h b/src/utf.h index 7a12dd99..21bc722b 100644 --- a/src/utf.h +++ b/src/utf.h @@ -45,7 +45,9 @@ static inline void MJB_USED mjb_diagnose_malformed_input(const char *buffer, siz mjb_encoding encoding, mjb_status status, mjb_diagnostic *diagnostic) { if(status == MJB_STATUS_MALFORMED_INPUT && diagnostic != NULL && diagnostic->error == MJB_TEXT_ERROR_NONE) { - (void)mjb_string_validate(buffer, byte_length, encoding, diagnostic); + mjb_status validation_status = mjb_string_validate(buffer, byte_length, encoding, + diagnostic); + (void)validation_status; } } From f5aecd099eaf6b2b2366c45b2b7e0a4ce7eeffbd Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Sat, 22 Aug 2026 23:24:59 +0200 Subject: [PATCH 3/7] Fix: warning C4310 --- tests/terminal-width.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/terminal-width.c b/tests/terminal-width.c index fa403e44..b257d8cc 100644 --- a/tests/terminal-width.c +++ b/tests/terminal-width.c @@ -162,8 +162,8 @@ int test_terminal_width(void *arg) { MJB_STATUS_MALFORMED_INPUT, "Malformed UTF-8") ATT_ASSERT(diagnostic.byte_offset, (size_t)0, "Terminal width malformed offset") - const char recoverable_width[] = { 'A', (char)0x80, (char)0xE4, (char)0xB8, (char)0xAD }; - ATT_ASSERT_STATUS(mjb_terminal_width(recoverable_width, sizeof(recoverable_width), + const unsigned char recoverable_width[] = { 'A', 0x80, 0xE4, 0xB8, 0xAD }; + ATT_ASSERT_STATUS(mjb_terminal_width((const char *)recoverable_width, sizeof(recoverable_width), MJB_ENC_UTF_8, MJB_MALFORMED_REPLACE, MJB_TERMINAL_WIDTH_NARROW, &sw, &diagnostic), MJB_STATUS_OK, "Terminal width replaces malformed input") @@ -171,7 +171,7 @@ int test_terminal_width(void *arg) { ATT_ASSERT(diagnostic.byte_offset, (size_t)1, "Terminal width replacement retains malformed offset") - ATT_ASSERT_STATUS(mjb_terminal_width(recoverable_width, sizeof(recoverable_width), + ATT_ASSERT_STATUS(mjb_terminal_width((const char *)recoverable_width, sizeof(recoverable_width), MJB_ENC_UTF_8, MJB_MALFORMED_SKIP, MJB_TERMINAL_WIDTH_NARROW, &sw, &diagnostic), MJB_STATUS_OK, "Terminal width skips malformed input") From ccc39931a9c1b4e647c3beeffcd0d69c97e65e26 Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Sat, 22 Aug 2026 23:36:24 +0200 Subject: [PATCH 4/7] Fix: missing field --- utils/generate/functions.ts | 2 +- utils/generate/html-function.ts | 5 ++++- utils/generate/types.ts | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/utils/generate/functions.ts b/utils/generate/functions.ts index 32b3c5a5..e9958434 100644 --- a/utils/generate/functions.ts +++ b/utils/generate/functions.ts @@ -129,7 +129,7 @@ function diagnostic(description = 'Where to store the first malformed-input diag name: 'diagnostic', type: 'mjb_diagnostic *', description, - wasm_generated: false + wasm_generated: true }; } diff --git a/utils/generate/html-function.ts b/utils/generate/html-function.ts index df08996c..99f5a18b 100644 --- a/utils/generate/html-function.ts +++ b/utils/generate/html-function.ts @@ -12,7 +12,7 @@ import functions, { import { caseModes, caseType, caseTypeValues, categories, collationStrengths, collationVariableWeightings, directions, encodings, encodingValues, filterFlags, filterFlagValues, - identifierProfiles, normalizations, planes, planeValues, statuses, terminalWidthProfiles + identifierProfiles, malformedPolicies, normalizations, planes, planeValues, statuses, terminalWidthProfiles } from './types'; const mojibakeTypes = new Set(); @@ -419,6 +419,9 @@ export class CFunction implements MojibakeFunction { case 'mjb_locale': ret += this.getInput(i); break; + case 'mjb_malformed_policy': + ret += this.getSelectInput(i, malformedPolicies); + break; // case 'mjb_locale_id *': case 'mjb_map_case_type': ret += this.getSelectInput(i, caseType, caseTypeValues); diff --git a/utils/generate/types.ts b/utils/generate/types.ts index 337b4fbc..6f4cd225 100644 --- a/utils/generate/types.ts +++ b/utils/generate/types.ts @@ -328,6 +328,12 @@ export const caseTypeValues = [ 4, ]; +export const malformedPolicies = [ + 'MJB_MALFORMED_STOP', + 'MJB_MALFORMED_REPLACE', + 'MJB_MALFORMED_SKIP', +]; + export const normalizations = [ 'MJB_NORMALIZATION_NFC', 'MJB_NORMALIZATION_NFD', From 8e17073901ff38a9047235d456b24e68e5af96cf Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Sat, 22 Aug 2026 23:45:43 +0200 Subject: [PATCH 5/7] Fix: GitHub security alert --- src/encoding.c | 44 ++++++++++++++++++++++++------------------- src/normalization.c | 10 ++-------- src/string.c | 7 ++++++- tests/normalization.c | 2 ++ tests/string.c | 9 +++++++++ 5 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/encoding.c b/src/encoding.c index eeecad69..26517509 100644 --- a/src/encoding.c +++ b/src/encoding.c @@ -217,11 +217,13 @@ static mjb_status mjb_decode_next_raw(const char *buffer, size_t byte_length, mj return MJB_STATUS_MALFORMED_INPUT; } - if(byte_length - start < 2) { + size_t remaining = byte_length - start; + + if(remaining < 2) { *offset = byte_length; *codepoint = MJB_CODEPOINT_REPLACEMENT; - mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, start, - byte_length - start, encoding); + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, start, remaining, + encoding); return MJB_STATUS_MALFORMED_INPUT; } @@ -256,17 +258,31 @@ static mjb_status mjb_decode_next_raw(const char *buffer, size_t byte_length, mj return MJB_STATUS_MALFORMED_INPUT; } + mjb_codepoint decoded; + + if(required == 2) { + decoded = (mjb_codepoint)(first & 0x1F); + } else if(required == 3) { + decoded = (mjb_codepoint)(first & 0x0F); + } else { + decoded = (mjb_codepoint)(first & 0x07); + } + + decoded = (decoded << 6) | ((mjb_codepoint)second & 0x3F); + for(size_t i = 2; i < required; ++i) { - if(start + i >= byte_length) { + if(i >= remaining) { *offset = byte_length; *codepoint = MJB_CODEPOINT_REPLACEMENT; - mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, start, - byte_length - start, encoding); + mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_TRUNCATED_SEQUENCE, start, remaining, + encoding); return MJB_STATUS_MALFORMED_INPUT; } - if(((uint8_t)buffer[start + i] & 0xC0) != 0x80) { + uint8_t continuation = (uint8_t)buffer[start + i]; + + if((continuation & 0xC0) != 0x80) { *offset = start + i; *codepoint = MJB_CODEPOINT_REPLACEMENT; mjb_set_diagnostic(diagnostic, MJB_TEXT_ERROR_MISSING_CONTINUATION, start, i, @@ -274,21 +290,11 @@ static mjb_status mjb_decode_next_raw(const char *buffer, size_t byte_length, mj return MJB_STATUS_MALFORMED_INPUT; } - } - if(required == 2) { - *codepoint = ((mjb_codepoint)(first & 0x1F) << 6) | ((mjb_codepoint)second & 0x3F); - } else if(required == 3) { - *codepoint = ((mjb_codepoint)(first & 0x0F) << 12) | - (((mjb_codepoint)second & 0x3F) << 6) | - ((mjb_codepoint)(uint8_t)buffer[start + 2] & 0x3F); - } else { - *codepoint = ((mjb_codepoint)(first & 0x07) << 18) | - (((mjb_codepoint)second & 0x3F) << 12) | - (((mjb_codepoint)(uint8_t)buffer[start + 2] & 0x3F) << 6) | - ((mjb_codepoint)(uint8_t)buffer[start + 3] & 0x3F); + decoded = (decoded << 6) | ((mjb_codepoint)continuation & 0x3F); } + *codepoint = decoded; *offset = start + required; return MJB_STATUS_OK; diff --git a/src/normalization.c b/src/normalization.c index a5f890c1..51953da8 100644 --- a/src/normalization.c +++ b/src/normalization.c @@ -817,15 +817,9 @@ static mjb_status mjb_nfkc_casefold_pass(const char *buffer, size_t byte_length, } if(output_index >= capacity) { - char *new_folded = (char *)mjb_realloc(folded, output_index + 1); + mjb_free(folded); - if(new_folded == NULL) { - mjb_free(folded); - - return MJB_STATUS_NO_MEMORY; - } - - folded = new_folded; + return MJB_STATUS_OVERFLOW; } folded[output_index] = '\0'; diff --git a/src/string.c b/src/string.c index 88454280..20bb7c4e 100644 --- a/src/string.c +++ b/src/string.c @@ -29,9 +29,14 @@ char *mjb_string_output(char *ret, char *input, size_t input_size, size_t *outpu return NULL; } + // Leave room for the terminator without allowing either addition below to wrap. + if(input_size >= SIZE_MAX - *output_index) { + return NULL; + } + if(*output_index + input_size >= *output_size) { size_t required = *output_index + input_size + 1; - size_t doubled = *output_size * 2; + size_t doubled = *output_size > SIZE_MAX / 2 ? SIZE_MAX : *output_size * 2; size_t new_output_size = (doubled > required) ? doubled : required; char *new_ret = (char *)mjb_realloc(ret, new_output_size); diff --git a/tests/normalization.c b/tests/normalization.c index 64706ccc..cbf4fde3 100644 --- a/tests/normalization.c +++ b/tests/normalization.c @@ -232,6 +232,8 @@ static void test_nfkc_casefold(void) { check_nfkc_casefold("A\xCC\x8A", 3, "\xC3\xA5", 2, "NFKC casefold composes mappings to NFC"); check_nfkc_casefold("\xE2\x84\xAA", 3, "k", 1, "NFKC casefold applies compatibility case mapping"); + check_nfkc_casefold("\xF0\x9F\x98\x80", 4, "\xF0\x9F\x98\x80", 4, + "NFKC casefold safely preserves a supplementary codepoint"); ATT_ASSERT_STATUS(mjb_nfkc_casefold("A", 1, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_ENC_UTF_16LE, &result, NULL), MJB_STATUS_OK, "NFKC casefold supports UTF-16 output") diff --git a/tests/string.c b/tests/string.c index def15b9a..926a373d 100644 --- a/tests/string.c +++ b/tests/string.c @@ -61,6 +61,15 @@ int test_string(void *arg) { ATT_ASSERT(mjb_string_output(NULL, output_input, 1, &output_index, NULL), (char *)NULL, "String output rejects NULL output size") + char overflow_output[1] = { 0 }; + output_index = SIZE_MAX; + output_size = sizeof(overflow_output); + ATT_ASSERT(mjb_string_output(overflow_output, output_input, 1, &output_index, &output_size), + (char *)NULL, "String output rejects an overflowing index") + output_index = 0; + ATT_ASSERT(mjb_string_output(overflow_output, output_input, SIZE_MAX, &output_index, &output_size), + (char *)NULL, "String output rejects an overflowing input size") + size_t into_size = 0; ATT_ASSERT_STATUS(mjb_output_into(NULL, &into_size, test_output_writer, NULL), MJB_STATUS_OK, "Output sink measures a transformation") From 2015bd46a875b1ee13fe3ee1024cb2af37c523e3 Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Wed, 26 Aug 2026 09:29:04 +0200 Subject: [PATCH 6/7] Remove useless members --- src/break-word.c | 2 -- src/mojibake.h | 2 -- src/segmentation.c | 1 - 3 files changed, 5 deletions(-) diff --git a/src/break-word.c b/src/break-word.c index f0b9bc07..611b31f7 100644 --- a/src/break-word.c +++ b/src/break-word.c @@ -66,7 +66,6 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le state->state = MJB_UTF_ACCEPT; state->previous = MJB_WBP_NOT_SET; state->current = MJB_WBP_NOT_SET; - state->prev_prev_codepoint = MJB_CODEPOINT_NOT_VALID; state->previous_codepoint = MJB_CODEPOINT_NOT_VALID; state->current_codepoint = MJB_CODEPOINT_NOT_VALID; state->prev_prev_wbp = MJB_WBP_NOT_SET; @@ -161,7 +160,6 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le // like WB7 and WB11 see the correct context before the WB4 base). if(!state->wb4_merged) { state->prev_prev_wbp = state->previous; - state->prev_prev_codepoint = state->previous_codepoint; } state->wb4_merged = false; diff --git a/src/mojibake.h b/src/mojibake.h index c38f3b78..2e441eb1 100644 --- a/src/mojibake.h +++ b/src/mojibake.h @@ -503,7 +503,6 @@ typedef struct mjb_next_state { unsigned short ri_count; bool ext_pict_seen; bool zwj_seen; - bool incb_consonant_seen; bool incb_linker_seen; } mjb_next_state; @@ -533,7 +532,6 @@ typedef struct mjb_next_word_state { size_t index; mjb_wbp previous; mjb_wbp current; - mjb_codepoint prev_prev_codepoint; mjb_codepoint previous_codepoint; mjb_codepoint current_codepoint; mjb_wbp prev_prev_wbp; diff --git a/src/segmentation.c b/src/segmentation.c index 31aa3731..6846164f 100644 --- a/src/segmentation.c +++ b/src/segmentation.c @@ -57,7 +57,6 @@ MJB_EXPORT mjb_break_type mjb_next_grapheme_break(const char *buffer, size_t byt state->ri_count = 0; state->ext_pict_seen = false; state->zwj_seen = false; - state->incb_consonant_seen = false; state->incb_linker_seen = false; } From 9b7d0bbaa9b1a1d625fecb3ab2f5a89c5d131c96 Mon Sep 17 00:00:00 2001 From: Francesco Bigiarini Date: Wed, 26 Aug 2026 09:40:48 +0200 Subject: [PATCH 7/7] Reorder members --- src/break-line.c | 8 ++++---- src/break-sentence.c | 2 +- src/break-word.c | 2 +- src/mojibake.h | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/break-line.c b/src/break-line.c index 62800fb0..c27d61ea 100644 --- a/src/break-line.c +++ b/src/break-line.c @@ -103,13 +103,13 @@ MJB_EXPORT mjb_break_type mjb_next_line_break(const char *buffer, size_t byte_le state->current_codepoint = MJB_CODEPOINT_NOT_VALID; state->in_error = false; state->ri_count = 0; - state->zw_seen = false; + state->prev_prev_codepoint = MJB_CODEPOINT_NOT_VALID; + state->prev_prev_lbp = MJB_LBP_NOT_SET; state->prev_resolved = MJB_LBP_NOT_SET; + state->prev_num_lbp = MJB_LBP_NOT_SET; state->prev_ea = MJB_EAW_NOT_SET; state->qu_prev_ea = MJB_EAW_NOT_SET; - state->prev_prev_lbp = MJB_LBP_NOT_SET; - state->prev_num_lbp = MJB_LBP_NOT_SET; - state->prev_prev_codepoint = MJB_CODEPOINT_NOT_VALID; + state->zw_seen = false; state->pi_qu_context = false; state->cm_merged = false; state->zwj_absorbed = false; diff --git a/src/break-sentence.c b/src/break-sentence.c index 8c583c3e..8e40a8d6 100644 --- a/src/break-sentence.c +++ b/src/break-sentence.c @@ -85,11 +85,11 @@ MJB_EXPORT mjb_break_type mjb_next_sentence_break(const char *buffer, size_t byt state->state = MJB_UTF_ACCEPT; state->previous = MJB_SBP_NOT_SET; state->current = MJB_SBP_NOT_SET; - state->prev_prev = MJB_SBP_NOT_SET; state->previous_codepoint = MJB_CODEPOINT_NOT_VALID; state->current_codepoint = MJB_CODEPOINT_NOT_VALID; state->in_error = false; state->had_error = false; + state->prev_prev = MJB_SBP_NOT_SET; state->sb5_merged = false; state->in_sat = false; state->sat_has_sp = false; diff --git a/src/break-word.c b/src/break-word.c index 611b31f7..2d164b55 100644 --- a/src/break-word.c +++ b/src/break-word.c @@ -68,10 +68,10 @@ MJB_EXPORT mjb_break_type mjb_next_word_break(const char *buffer, size_t byte_le state->current = MJB_WBP_NOT_SET; state->previous_codepoint = MJB_CODEPOINT_NOT_VALID; state->current_codepoint = MJB_CODEPOINT_NOT_VALID; - state->prev_prev_wbp = MJB_WBP_NOT_SET; state->in_error = false; state->had_error = false; state->ri_count = 0; + state->prev_prev_wbp = MJB_WBP_NOT_SET; state->wb4_merged = false; state->zwj_pending = false; state->prev_was_zwj = false; diff --git a/src/mojibake.h b/src/mojibake.h index 2e441eb1..64fee977 100644 --- a/src/mojibake.h +++ b/src/mojibake.h @@ -501,6 +501,7 @@ typedef struct mjb_next_state { bool in_error; bool had_error; unsigned short ri_count; + // Grapheme break-specific members bool ext_pict_seen; bool zwj_seen; bool incb_linker_seen; @@ -511,20 +512,21 @@ typedef struct mjb_next_line_state { size_t index; mjb_lbp previous; mjb_lbp current; - mjb_codepoint prev_prev_codepoint; mjb_codepoint previous_codepoint; mjb_codepoint current_codepoint; bool in_error; - bool zw_seen; - bool pi_qu_context; - bool cm_merged; - bool zwj_absorbed; unsigned short ri_count; + // Line break-specific members + mjb_codepoint prev_prev_codepoint; + mjb_lbp prev_prev_lbp; mjb_lbp prev_resolved; + mjb_lbp prev_num_lbp; mjb_east_asian_width prev_ea; mjb_east_asian_width qu_prev_ea; - mjb_lbp prev_prev_lbp; - mjb_lbp prev_num_lbp; + bool zw_seen; + bool pi_qu_context; + bool cm_merged; + bool zwj_absorbed; } mjb_next_line_state; typedef struct mjb_next_word_state { @@ -534,10 +536,11 @@ typedef struct mjb_next_word_state { mjb_wbp current; mjb_codepoint previous_codepoint; mjb_codepoint current_codepoint; - mjb_wbp prev_prev_wbp; bool in_error; bool had_error; unsigned short ri_count; + // Word break-specific members + mjb_wbp prev_prev_wbp; bool wb4_merged; bool zwj_pending; bool prev_was_zwj; @@ -548,11 +551,12 @@ typedef struct mjb_next_sentence_state { size_t index; mjb_sbp previous; mjb_sbp current; - mjb_sbp prev_prev; mjb_codepoint previous_codepoint; mjb_codepoint current_codepoint; bool in_error; bool had_error; + // Sentence break-specific members + mjb_sbp prev_prev; bool sb5_merged; bool in_sat; bool sat_has_sp;