Add malformed policy

CONTRIBUTING.md

@@ -192,8 +192,9 @@ new assertion:
 
 ```c
 size_t count = 0;
-ATT_ASSERT_STATUS(mjb_codepoint_count("Hello, test", 11, MJB_ENC_UTF_8, &count), MJB_STATUS_OK,
-    "UTF-8 Hello, test status")
+ATT_ASSERT_STATUS(mjb_codepoint_count("Hello, test", 11, MJB_ENC_UTF_8, MJB_MALFORMED_STOP,
+                      &count, NULL),
+    MJB_STATUS_OK, "UTF-8 Hello, test status")
 ATT_ASSERT(count, (size_t)11, "UTF-8 Hello, test")
 ```
 

README.md

@@ -27,8 +27,8 @@ int main(int argc, char *const argv[]) {
     mjb_result result;
 
     // Normalize example: in NFC e + ◌́ -> é (U+00E9)
-    if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8,
-        &result) != MJB_STATUS_OK) {
+    if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP,
+        MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) {
         return 1;
     }
 
@@ -44,7 +44,8 @@ int main(int argc, char *const argv[]) {
     // Codepoint count example: mjb_codepoint_count counts Unicode codepoints, not bytes.
     size_t codepoint_count = 0;
 
-    if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, &codepoint_count) == MJB_STATUS_OK) {
+    if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP,
+           &codepoint_count, NULL) == MJB_STATUS_OK) {
         printf("\"%s\" encoded in UTF-8 is %zu bytes long, and %zu codepoints long\n",
             mojibake, length, codepoint_count);
     }
@@ -54,8 +55,8 @@ int main(int argc, char *const argv[]) {
     const char *case_input = "Straße";
 
     // NFKC casefold example: in NFKC casefold, ß -> ss
-    if(mjb_nfkc_casefold(case_input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8, MJB_ENC_UTF_8,
-        &result) != MJB_STATUS_OK) {
+    if(mjb_nfkc_casefold(case_input, MJB_NUL_TERMINATED, MJB_ENC_UTF_8,
+        MJB_MALFORMED_STOP, MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) {
         return 1;
     }
 
@@ -168,10 +169,11 @@ and header: `mojibake.c` and `mojibake.h`. Zero dependencies.
 **Integration**
 
 - **Encodings**: the API accepts and outputs UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE
-  strings, with encoding detection and conversion (`mjb_detect_encoding`,
-  `mjb_convert_encoding`, `mjb_convert_encoding_into`)
-- **Parsing and string functions**: codepoint-by-codepoint iteration (`mjb_for_each_codepoint`)
-  and codepoint counting (`mjb_codepoint_count`)
+  strings, with detailed validation and explicit stop, replacement, or skip policies during
+  conversion (`mjb_string_validate`, `mjb_convert_encoding`, `mjb_convert_encoding_into`)
+- **Parsing and string functions**: forward and reverse decoding with malformed-input diagnostics
+  (`mjb_decode_next`, `mjb_decode_previous`), codepoint callbacks (`mjb_for_each_codepoint`), and
+  policy-aware codepoint counting (`mjb_codepoint_count`)
 - **Locales**: strict BCP 47 language tag parsing (`mjb_locale_parse`)
 - **Embeddable**: context-aware custom allocators (`mjb_set_allocator`), build-time feature flags
   to trim table size, a C++17 wrapper (`src/cpp/mojibake.hpp`), a CLI tool (`src/shell`), and a

ROADMAP.md

@@ -7,20 +7,17 @@ This is the Mojibake roadmap. For great justice.
 1. **Complete UTS #39 identifier checks**
    Generate compact tables from `IdentifierStatus.txt` and `IdentifierType.txt`, then add
    `mjb_identifier_check` and `mjb_string_restriction_level`.
-2. **Detailed validation and decoder iteration**
-   Add `mjb_string_validate`, `mjb_decode_next`, and `mjb_decode_previous`. Use a shared diagnostic
-   result with the first failing byte (or code-unit offset?) and a precise malformed-input kind.
-3. **Finish typed UCD access**
+2. **Finish typed UCD access**
    Add typed getters for code-point, code-point-sequence, and string-valued properties. Follow with
    character age, bidi mirror, modern/alias/extended names, and reverse character-name lookup. Is
    this needed?
-4. **Explicit locale operations**
+3. **Explicit locale operations**
    Implement the currently unsupported `mjb_locale_canonicalize` using a pinned IANA Language
    Subtag Registry snapshot.
-5. **Reusable configurable collators**
+4. **Reusable configurable collators**
    Introduce immutable DUCET collator objects that encapsulate strength and variable weighting,
    then add case ordering, numeric collation, and normalization options.
-6. **Streaming processing**
+5. **Streaming processing**
    Add stateful `init`/`feed`/`finish` APIs for decoding and conversion, normalization, casing, and
    segmentation, using caller-kind of API.
 

examples/example.c

@@ -20,8 +20,8 @@ int main(int argc, char *const argv[]) {
     mjb_result result;
 
     // Normalize example: in NFC e + ◌́ -> é (U+00E9)
-    if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_NORMALIZATION_NFC, MJB_ENC_UTF_8, &result) !=
-        MJB_STATUS_OK) {
+    if(mjb_normalize(input, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, MJB_NORMALIZATION_NFC,
+           MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) {
         return 1;
     }
 
@@ -37,7 +37,8 @@ int main(int argc, char *const argv[]) {
     // Codepoint count example: mjb_codepoint_count counts Unicode codepoints, not bytes.
     size_t codepoint_count = 0;
 
-    if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, &codepoint_count) != MJB_STATUS_OK) {
+    if(mjb_codepoint_count(mojibake, length, MJB_ENC_UTF_8, MJB_MALFORMED_STOP, &codepoint_count,
+           NULL) != MJB_STATUS_OK) {
         return 1;
     }
 
@@ -49,8 +50,8 @@ int main(int argc, char *const argv[]) {
     const char *case_input = "Straße";
 
     // NFKC casefold example: in NFKC casefold, ß -> ss
-    if(mjb_nfkc_casefold(case_input, strlen(case_input), MJB_ENC_UTF_8, MJB_ENC_UTF_8, &result) !=
-        MJB_STATUS_OK) {
+    if(mjb_nfkc_casefold(case_input, strlen(case_input), MJB_ENC_UTF_8, MJB_MALFORMED_STOP,
+           MJB_ENC_UTF_8, &result, NULL) != MJB_STATUS_OK) {
         return 1;
     }
 

examples/python/example.py

@@ -16,6 +16,7 @@
 
 MJB_STATUS_OK = 0
 MJB_ENC_UTF_8 = 0x2
+MJB_MALFORMED_STOP = 0
 MJB_NORMALIZATION_NFC = 0
 
 
@@ -46,7 +47,9 @@ def load_mojibake() -> ctypes.CDLL:
         ctypes.c_int,
         ctypes.c_int,
         ctypes.c_int,
+        ctypes.c_int,
         ctypes.POINTER(MjbResult),
+        ctypes.c_void_p,
     ]
 
     library.mjb_normalize.restype = ctypes.c_int
@@ -56,7 +59,9 @@ def load_mojibake() -> ctypes.CDLL:
         ctypes.c_size_t,
         ctypes.c_int,
         ctypes.c_int,
+        ctypes.c_int,
         ctypes.POINTER(MjbResult),
+        ctypes.c_void_p,
     ]
 
     library.mjb_nfkc_casefold.restype = ctypes.c_int
@@ -65,7 +70,9 @@ def load_mojibake() -> ctypes.CDLL:
         ctypes.c_char_p,
         ctypes.c_size_t,
         ctypes.c_int,
+        ctypes.c_int,
         ctypes.POINTER(ctypes.c_size_t),
+        ctypes.c_void_p,
     ]
 
     library.mjb_codepoint_count.restype = ctypes.c_int
@@ -88,9 +95,11 @@ def normalize(library: ctypes.CDLL, input_bytes: bytes) -> bytes:
         input_bytes,
         len(input_bytes),
         MJB_ENC_UTF_8,
+        MJB_MALFORMED_STOP,
         MJB_NORMALIZATION_NFC,
         MJB_ENC_UTF_8,
         ctypes.byref(result),
+        None,
     )
 
     if status != MJB_STATUS_OK:
@@ -105,8 +114,10 @@ def nfkc_casefold(library: ctypes.CDLL, input_bytes: bytes) -> bytes:
         input_bytes,
         len(input_bytes),
         MJB_ENC_UTF_8,
+        MJB_MALFORMED_STOP,
         MJB_ENC_UTF_8,
         ctypes.byref(result),
+        None,
     )
 
     if status != MJB_STATUS_OK:
@@ -143,7 +154,7 @@ def main() -> int:
     # Codepoint count example: mjb_codepoint_count counts Unicode codepoints, not bytes.
     codepoint_count = ctypes.c_size_t(0)
     status = library.mjb_codepoint_count(
-        mojibake, len(mojibake), MJB_ENC_UTF_8, ctypes.byref(codepoint_count)
+        mojibake, len(mojibake), MJB_ENC_UTF_8, 0, ctypes.byref(codepoint_count), None
     )
 
     if status != MJB_STATUS_OK:

examples/rust/example.rs

@@ -9,6 +9,7 @@ use std::slice;
 const MJB_STATUS_OK: c_int = 0;
 const MJB_ENC_UTF_8: c_int = 0x2;
 const MJB_NORMALIZATION_NFC: c_int = 0;
+const MJB_MALFORMED_STOP: c_int = 0;
 
 #[repr(C)]
 struct MjbResult {
@@ -48,24 +49,30 @@ extern "C" {
         buffer: *const c_char,
         byte_length: usize,
         encoding: c_int,
+        malformed_policy: c_int,
         form: c_int,
         output_encoding: c_int,
         result: *mut MjbResult,
+        diagnostic: *mut std::ffi::c_void,
     ) -> c_int;
 
     fn mjb_nfkc_casefold(
         buffer: *const c_char,
         byte_length: usize,
         encoding: c_int,
+        malformed_policy: c_int,
         output_encoding: c_int,
         result: *mut MjbResult,
+        diagnostic: *mut std::ffi::c_void,
     ) -> c_int;
 
     fn mjb_codepoint_count(
         buffer: *const c_char,
         byte_length: usize,
         encoding: c_int,
+        malformed_policy: c_int,
         count: *mut usize,
+        diagnostic: *mut std::ffi::c_void,
     ) -> c_int;
 
     fn mjb_result_free(result: *mut MjbResult) -> c_int;
@@ -92,9 +99,11 @@ fn run() -> bool {
             input.as_ptr().cast(),
             input.len(),
             MJB_ENC_UTF_8,
+            MJB_MALFORMED_STOP,
             MJB_NORMALIZATION_NFC,
             MJB_ENC_UTF_8,
             &mut normalized,
+            std::ptr::null_mut(),
         )
     } != MJB_STATUS_OK
     {
@@ -116,7 +125,9 @@ fn run() -> bool {
             mojibake.as_ptr().cast(),
             mojibake.len(),
             MJB_ENC_UTF_8,
+            MJB_MALFORMED_STOP,
             &mut codepoint_count,
+            std::ptr::null_mut(),
         )
     };
     assert_eq!(status, MJB_STATUS_OK, "mjb_codepoint_count failed");
@@ -134,8 +145,10 @@ fn run() -> bool {
             case_input.as_ptr().cast(),
             case_input.len(),
             MJB_ENC_UTF_8,
+            MJB_MALFORMED_STOP,
             MJB_ENC_UTF_8,
             &mut casefolded,
+            std::ptr::null_mut(),
         )
     } != MJB_STATUS_OK
     {

examples/zig/example.zig

@@ -23,9 +23,11 @@ pub fn main(init: std.process.Init) !u8 {
             input.ptr,
             input.len,
             mjb.MJB_ENC_UTF_8,
+            mjb.MJB_MALFORMED_STOP,
             mjb.MJB_NORMALIZATION_NFC,
             mjb.MJB_ENC_UTF_8,
             &result,
+            null,
         ) != mjb.MJB_STATUS_OK) {
             return 1;
         }
@@ -46,7 +48,9 @@ pub fn main(init: std.process.Init) !u8 {
             mojibake.ptr,
             mojibake.len,
             mjb.MJB_ENC_UTF_8,
+            mjb.MJB_MALFORMED_STOP,
             &codepoint_count,
+            null,
         ) != mjb.MJB_STATUS_OK) {
             return error.CodepointCountFailed;
         }
@@ -66,8 +70,10 @@ pub fn main(init: std.process.Init) !u8 {
             case_input.ptr,
             case_input.len,
             mjb.MJB_ENC_UTF_8,
+            mjb.MJB_MALFORMED_STOP,
             mjb.MJB_ENC_UTF_8,
             &result,
+            null,
         ) != mjb.MJB_STATUS_OK) {
             return 1;
         }