Decode JSON UTF-16 surrogate pairs instead of emitting invalid UTF-8

CHANGELOG.md

@@ -8,6 +8,10 @@ loosely while pre-1.0 (breaking changes can land on minor bumps).
 ## [Unreleased]
 
 ### Fixed
+- **JSON UTF-16 surrogate pairs.** `json_parse` now decodes `\uD800`–`\uDFFF`
+  pairs into a single supplementary-plane scalar (emoji, etc.) instead of
+  two invalid 3-byte sequences that `json_serialize` then replaced with
+  U+FFFD. Unpaired surrogates become U+FFFD.
 - **TUI replay QUERY boundary.** `replay_user_echo_text` now strips the
   first `\n\nQUERY: ` after the AGENTS roster (not the last). User text
   that embeds that marker no longer replays as only the suffix after it.

CMakeLists.txt

@@ -1384,6 +1384,19 @@ if(INDEX_BUILD_TESTS)
     )
     add_test(NAME unit_logger COMMAND unit_logger)
 
+    # JSON \u escapes including UTF-16 surrogate pairs (emoji via
+    # Python json.dumps / MCP-A2A peers).  Own binary so it stays
+    # independently mergeable vs unit_mcp framing tests.
+    add_executable(unit_json
+        tests/test_json.cpp
+        src/json.cpp
+    )
+    target_include_directories(unit_json PRIVATE
+        ${CMAKE_SOURCE_DIR}/include
+        ${CMAKE_SOURCE_DIR}/third_party/doctest
+    )
+    add_test(NAME unit_json COMMAND unit_json)
+
     # --- Unit tests for the structured-memory store ---
     # Pins the tenant_store CRUD contract that backs both /v1/memory/entries
     # and the agent-side /mem entries|entry|search reader.

src/json.cpp

@@ -162,6 +162,44 @@ struct Parser {
             std::string("JSON parse error at pos ") + std::to_string(pos) + ": " + msg);
     }
 
+    unsigned parse_hex4() {
+        unsigned cp = 0;
+        for (int i = 0; i < 4; ++i) {
+            char h = advance();
+            cp <<= 4;
+            if (h >= '0' && h <= '9') cp |= static_cast<unsigned>(h - '0');
+            else if (h >= 'a' && h <= 'f') cp |= static_cast<unsigned>(h - 'a' + 10);
+            else if (h >= 'A' && h <= 'F') cp |= static_cast<unsigned>(h - 'A' + 10);
+            else error("invalid hex in \\u escape");
+        }
+        return cp;
+    }
+
+    // Encode a Unicode scalar as UTF-8.  Unpaired surrogates and values
+    // above U+10FFFF become U+FFFD — same replacement the serializer
+    // uses for malformed input bytes.
+    static void append_utf8(std::string& out, uint32_t cp) {
+        if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) {
+            out += "\xEF\xBF\xBD";
+            return;
+        }
+        if (cp < 0x80) {
+            out += static_cast<char>(cp);
+        } else if (cp < 0x800) {
+            out += static_cast<char>(0xC0 | (cp >> 6));
+            out += static_cast<char>(0x80 | (cp & 0x3F));
+        } else if (cp < 0x10000) {
+            out += static_cast<char>(0xE0 | (cp >> 12));
+            out += static_cast<char>(0x80 | ((cp >> 6) & 0x3F));
+            out += static_cast<char>(0x80 | (cp & 0x3F));
+        } else {
+            out += static_cast<char>(0xF0 | (cp >> 18));
+            out += static_cast<char>(0x80 | ((cp >> 12) & 0x3F));
+            out += static_cast<char>(0x80 | ((cp >> 6) & 0x3F));
+            out += static_cast<char>(0x80 | (cp & 0x3F));
+        }
+    }
+
     std::shared_ptr<JsonValue> parse_value() {
         skip_ws();
         char c = peek();
@@ -194,26 +232,37 @@ struct Parser {
                     case 'r':  result += '\r'; break;
                     case 't':  result += '\t'; break;
                     case 'u': {
-                        // Basic BMP only
-                        unsigned cp = 0;
-                        for (int i = 0; i < 4; ++i) {
-                            char h = advance();
-                            cp <<= 4;
-                            if (h >= '0' && h <= '9') cp |= (h - '0');
-                            else if (h >= 'a' && h <= 'f') cp |= (h - 'a' + 10);
-                            else if (h >= 'A' && h <= 'F') cp |= (h - 'A' + 10);
-                            else error("invalid hex in \\u escape");
-                        }
-                        if (cp < 0x80) {
-                            result += static_cast<char>(cp);
-                        } else if (cp < 0x800) {
-                            result += static_cast<char>(0xC0 | (cp >> 6));
-                            result += static_cast<char>(0x80 | (cp & 0x3F));
-                        } else {
-                            result += static_cast<char>(0xE0 | (cp >> 12));
-                            result += static_cast<char>(0x80 | ((cp >> 6) & 0x3F));
-                            result += static_cast<char>(0x80 | (cp & 0x3F));
+                        // RFC 8259 §7: non-BMP scalars are a UTF-16
+                        // surrogate pair.  Python json.dumps (default
+                        // ensure_ascii=True) and many MCP/A2A peers emit
+                        // that form for emoji.  Encoding each half as a
+                        // 3-byte sequence is not UTF-8; escape_string then
+                        // replaces those bytes with U+FFFD on re-serialize.
+                        unsigned cp = parse_hex4();
+                        if (cp >= 0xD800 && cp <= 0xDBFF) {
+                            if (pos + 6 <= src.size() &&
+                                src[pos] == '\\' && src[pos + 1] == 'u') {
+                                advance();
+                                advance();
+                                unsigned low = parse_hex4();
+                                if (low >= 0xDC00 && low <= 0xDFFF) {
+                                    cp = 0x10000u +
+                                         ((cp - 0xD800u) << 10) +
+                                         (low - 0xDC00u);
+                                } else {
+                                    append_utf8(result, 0xFFFD);
+                                    append_utf8(result, low);
+                                    break;
+                                }
+                            } else {
+                                append_utf8(result, 0xFFFD);
+                                break;
+                            }
+                        } else if (cp >= 0xDC00 && cp <= 0xDFFF) {
+                            append_utf8(result, 0xFFFD);
+                            break;
                         }
+                        append_utf8(result, cp);
                         break;
                     }
                     default: error("invalid escape");

tests/test_json.cpp

@@ -0,0 +1,78 @@
+// tests/test_json.cpp — JSON string \u escapes, including UTF-16
+// surrogate pairs.  Isolated from unit_mcp so this stays independently
+// mergeable against open MCP framing PRs.
+
+#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+#include "doctest.h"
+
+#include "json.h"
+
+using namespace arbiter;
+
+TEST_CASE("json_parse BMP \\u escapes") {
+    auto v = json_parse(R"("caf\u00e9")");
+    REQUIRE(v);
+    REQUIRE(v->is_string());
+    CHECK(v->as_string() == "café");
+}
+
+TEST_CASE("json_parse UTF-16 surrogate pair is one scalar") {
+    // U+1F600 GRINNING FACE — Python json.dumps("😀") default form.
+    auto v = json_parse(R"("\uD83D\uDE00")");
+    REQUIRE(v);
+    REQUIRE(v->is_string());
+    CHECK(v->as_string() == "😀");
+    CHECK(v->as_string().size() == 4);
+    CHECK(static_cast<unsigned char>(v->as_string()[0]) == 0xF0);
+    CHECK(static_cast<unsigned char>(v->as_string()[1]) == 0x9F);
+    CHECK(static_cast<unsigned char>(v->as_string()[2]) == 0x98);
+    CHECK(static_cast<unsigned char>(v->as_string()[3]) == 0x80);
+}
+
+TEST_CASE("json_parse lowercase surrogate pair") {
+    auto v = json_parse(R"("\ud83d\ude00")");
+    REQUIRE(v);
+    CHECK(v->as_string() == "😀");
+}
+
+TEST_CASE("json_serialize of a decoded pair is well-formed UTF-8 JSON") {
+    auto v = json_parse(R"("\uD83D\uDE00")");
+    REQUIRE(v);
+    const std::string wire = json_serialize(*v);
+    CHECK(wire == "\"😀\"");
+    auto round = json_parse(wire);
+    REQUIRE(round);
+    CHECK(round->as_string() == "😀");
+}
+
+TEST_CASE("json_parse unpaired high surrogate becomes U+FFFD") {
+    auto v = json_parse(R"("\uD83D")");
+    REQUIRE(v);
+    CHECK(v->as_string() == "\xEF\xBF\xBD");
+}
+
+TEST_CASE("json_parse unpaired low surrogate becomes U+FFFD") {
+    auto v = json_parse(R"("\uDE00")");
+    REQUIRE(v);
+    CHECK(v->as_string() == "\xEF\xBF\xBD");
+}
+
+TEST_CASE("json_parse high surrogate followed by BMP keeps the BMP") {
+    auto v = json_parse(R"("\uD83D\u00e9")");
+    REQUIRE(v);
+    CHECK(v->as_string() == "\xEF\xBF\xBD" "é");
+}
+
+TEST_CASE("json_parse nested object with escaped emoji (MCP/A2A shape)") {
+    auto v = json_parse(
+        R"({"result":{"content":[{"type":"text","text":"hi \uD83D\uDE00"}]}})");
+    REQUIRE(v);
+    auto text = v->get("result")->get("content")->as_array().at(0)->get("text");
+    REQUIRE(text);
+    CHECK(text->as_string() == "hi 😀");
+}
+
+TEST_CASE("json_parse rejects truncated \\u hex") {
+    CHECK_THROWS(json_parse(R"("\uD8")"));
+    CHECK_THROWS(json_parse(R"("\uD83D\uDE0")"));
+}