diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c6d65a..fd86528e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ loosely while pre-1.0 (breaking changes can land on minor bumps). ## [Unreleased] ### Fixed +- **JSON UTF-16 surrogate pairs.** `json_parse` now decodes `\uD800`–`\uDFFF` + pairs into a single supplementary-plane scalar (emoji, etc.) instead of + two invalid 3-byte sequences that `json_serialize` then replaced with + U+FFFD. Unpaired surrogates become U+FFFD. - **TUI replay QUERY boundary.** `replay_user_echo_text` now strips the first `\n\nQUERY: ` after the AGENTS roster (not the last). User text that embeds that marker no longer replays as only the suffix after it. diff --git a/CMakeLists.txt b/CMakeLists.txt index bb999ccb..071baa9d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1384,6 +1384,19 @@ if(INDEX_BUILD_TESTS) ) add_test(NAME unit_logger COMMAND unit_logger) + # JSON \u escapes including UTF-16 surrogate pairs (emoji via + # Python json.dumps / MCP-A2A peers). Own binary so it stays + # independently mergeable vs unit_mcp framing tests. + add_executable(unit_json + tests/test_json.cpp + src/json.cpp + ) + target_include_directories(unit_json PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/third_party/doctest + ) + add_test(NAME unit_json COMMAND unit_json) + # --- Unit tests for the structured-memory store --- # Pins the tenant_store CRUD contract that backs both /v1/memory/entries # and the agent-side /mem entries|entry|search reader. diff --git a/src/json.cpp b/src/json.cpp index 57c9aef4..33636f5e 100644 --- a/src/json.cpp +++ b/src/json.cpp @@ -162,6 +162,44 @@ struct Parser { std::string("JSON parse error at pos ") + std::to_string(pos) + ": " + msg); } + unsigned parse_hex4() { + unsigned cp = 0; + for (int i = 0; i < 4; ++i) { + char h = advance(); + cp <<= 4; + if (h >= '0' && h <= '9') cp |= static_cast(h - '0'); + else if (h >= 'a' && h <= 'f') cp |= static_cast(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') cp |= static_cast(h - 'A' + 10); + else error("invalid hex in \\u escape"); + } + return cp; + } + + // Encode a Unicode scalar as UTF-8. Unpaired surrogates and values + // above U+10FFFF become U+FFFD — same replacement the serializer + // uses for malformed input bytes. + static void append_utf8(std::string& out, uint32_t cp) { + if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) { + out += "\xEF\xBF\xBD"; + return; + } + if (cp < 0x80) { + out += static_cast(cp); + } else if (cp < 0x800) { + out += static_cast(0xC0 | (cp >> 6)); + out += static_cast(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + out += static_cast(0xE0 | (cp >> 12)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } else { + out += static_cast(0xF0 | (cp >> 18)); + out += static_cast(0x80 | ((cp >> 12) & 0x3F)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } + } + std::shared_ptr parse_value() { skip_ws(); char c = peek(); @@ -194,26 +232,37 @@ struct Parser { case 'r': result += '\r'; break; case 't': result += '\t'; break; case 'u': { - // Basic BMP only - unsigned cp = 0; - for (int i = 0; i < 4; ++i) { - char h = advance(); - cp <<= 4; - if (h >= '0' && h <= '9') cp |= (h - '0'); - else if (h >= 'a' && h <= 'f') cp |= (h - 'a' + 10); - else if (h >= 'A' && h <= 'F') cp |= (h - 'A' + 10); - else error("invalid hex in \\u escape"); - } - if (cp < 0x80) { - result += static_cast(cp); - } else if (cp < 0x800) { - result += static_cast(0xC0 | (cp >> 6)); - result += static_cast(0x80 | (cp & 0x3F)); - } else { - result += static_cast(0xE0 | (cp >> 12)); - result += static_cast(0x80 | ((cp >> 6) & 0x3F)); - result += static_cast(0x80 | (cp & 0x3F)); + // RFC 8259 §7: non-BMP scalars are a UTF-16 + // surrogate pair. Python json.dumps (default + // ensure_ascii=True) and many MCP/A2A peers emit + // that form for emoji. Encoding each half as a + // 3-byte sequence is not UTF-8; escape_string then + // replaces those bytes with U+FFFD on re-serialize. + unsigned cp = parse_hex4(); + if (cp >= 0xD800 && cp <= 0xDBFF) { + if (pos + 6 <= src.size() && + src[pos] == '\\' && src[pos + 1] == 'u') { + advance(); + advance(); + unsigned low = parse_hex4(); + if (low >= 0xDC00 && low <= 0xDFFF) { + cp = 0x10000u + + ((cp - 0xD800u) << 10) + + (low - 0xDC00u); + } else { + append_utf8(result, 0xFFFD); + append_utf8(result, low); + break; + } + } else { + append_utf8(result, 0xFFFD); + break; + } + } else if (cp >= 0xDC00 && cp <= 0xDFFF) { + append_utf8(result, 0xFFFD); + break; } + append_utf8(result, cp); break; } default: error("invalid escape"); diff --git a/tests/test_json.cpp b/tests/test_json.cpp new file mode 100644 index 00000000..57ea1d29 --- /dev/null +++ b/tests/test_json.cpp @@ -0,0 +1,78 @@ +// tests/test_json.cpp — JSON string \u escapes, including UTF-16 +// surrogate pairs. Isolated from unit_mcp so this stays independently +// mergeable against open MCP framing PRs. + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "doctest.h" + +#include "json.h" + +using namespace arbiter; + +TEST_CASE("json_parse BMP \\u escapes") { + auto v = json_parse(R"("caf\u00e9")"); + REQUIRE(v); + REQUIRE(v->is_string()); + CHECK(v->as_string() == "café"); +} + +TEST_CASE("json_parse UTF-16 surrogate pair is one scalar") { + // U+1F600 GRINNING FACE — Python json.dumps("😀") default form. + auto v = json_parse(R"("\uD83D\uDE00")"); + REQUIRE(v); + REQUIRE(v->is_string()); + CHECK(v->as_string() == "😀"); + CHECK(v->as_string().size() == 4); + CHECK(static_cast(v->as_string()[0]) == 0xF0); + CHECK(static_cast(v->as_string()[1]) == 0x9F); + CHECK(static_cast(v->as_string()[2]) == 0x98); + CHECK(static_cast(v->as_string()[3]) == 0x80); +} + +TEST_CASE("json_parse lowercase surrogate pair") { + auto v = json_parse(R"("\ud83d\ude00")"); + REQUIRE(v); + CHECK(v->as_string() == "😀"); +} + +TEST_CASE("json_serialize of a decoded pair is well-formed UTF-8 JSON") { + auto v = json_parse(R"("\uD83D\uDE00")"); + REQUIRE(v); + const std::string wire = json_serialize(*v); + CHECK(wire == "\"😀\""); + auto round = json_parse(wire); + REQUIRE(round); + CHECK(round->as_string() == "😀"); +} + +TEST_CASE("json_parse unpaired high surrogate becomes U+FFFD") { + auto v = json_parse(R"("\uD83D")"); + REQUIRE(v); + CHECK(v->as_string() == "\xEF\xBF\xBD"); +} + +TEST_CASE("json_parse unpaired low surrogate becomes U+FFFD") { + auto v = json_parse(R"("\uDE00")"); + REQUIRE(v); + CHECK(v->as_string() == "\xEF\xBF\xBD"); +} + +TEST_CASE("json_parse high surrogate followed by BMP keeps the BMP") { + auto v = json_parse(R"("\uD83D\u00e9")"); + REQUIRE(v); + CHECK(v->as_string() == "\xEF\xBF\xBD" "é"); +} + +TEST_CASE("json_parse nested object with escaped emoji (MCP/A2A shape)") { + auto v = json_parse( + R"({"result":{"content":[{"type":"text","text":"hi \uD83D\uDE00"}]}})"); + REQUIRE(v); + auto text = v->get("result")->get("content")->as_array().at(0)->get("text"); + REQUIRE(text); + CHECK(text->as_string() == "hi 😀"); +} + +TEST_CASE("json_parse rejects truncated \\u hex") { + CHECK_THROWS(json_parse(R"("\uD8")")); + CHECK_THROWS(json_parse(R"("\uD83D\uDE0")")); +}