From d5bace24cd827e202a290b41d9a7e1d2ed7d9fab Mon Sep 17 00:00:00 2001 From: romer8 Date: Wed, 20 May 2026 20:29:59 -0600 Subject: [PATCH] fix(query): normalize NaN cells to None in query_files_by_selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit df.to_dict(orient="records") preserves pandas NaN as float("nan"), which Python's default json.dumps(allow_nan=True) emits as the literal token "NaN" — not valid JSON. JS clients that JSON.parse the result silently fall through to a raw-string path. In chatbox-core that bypasses both _engine_dispatched and _cache_uri injection (engine/index.js:1060 gates on isObjResult, which is false for strings). Affected every hydrology query result that touched the troute "nudge" column on a non-assimilated reach. The sibling code path at logic.py:754 already applied _normalize_record for the same reason. This brings line 649 in line with it. --- nextgen_mcp/logic.py | 2 +- test_mcp/test_query_files_by_selector.py | 49 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/nextgen_mcp/logic.py b/nextgen_mcp/logic.py index ebd9c32..ca504d5 100644 --- a/nextgen_mcp/logic.py +++ b/nextgen_mcp/logic.py @@ -646,7 +646,7 @@ def query_files_by_selector( query=query, columns=list(df.columns), rows=int(len(df)), - data=df.to_dict(orient="records"), + data=[_normalize_record(r) for r in df.to_dict(orient="records")], ) # Surface the exclusion count when non-zero so the user/LLM has a diff --git a/test_mcp/test_query_files_by_selector.py b/test_mcp/test_query_files_by_selector.py index 6f4259b..2052e1d 100644 --- a/test_mcp/test_query_files_by_selector.py +++ b/test_mcp/test_query_files_by_selector.py @@ -148,6 +148,55 @@ def test_index_filter_returns_single_file_at_index(monkeypatch): assert captured["urls"][0].endswith("b.parquet") +# --------------------------------------------------------------------------- +# JSON-serialization contract +# --------------------------------------------------------------------------- + + +def test_nan_cells_serialize_to_valid_json(monkeypatch): + """NaN cells in the query result MUST become JSON null, not literal NaN. + + Backstory: troute parquet output contains NaN in the ``nudge`` column for + non-assimilated reaches. Without normalization, pandas keeps the value as + ``float('nan')`` and Python's default ``json.dumps`` emits literal ``NaN``, + which (a) is not valid JSON and (b) makes the entire result envelope + un-parseable in JS clients. In chatbox-core, that string-failure path + bypasses ``_engine_dispatched`` + ``_cache_uri`` injection silently — so + the LLM tokenizes the full payload into the next tool call instead of + referencing the cache. Lock the contract: the envelope round-trips + through strict ``json.loads(json.dumps(...))``. + """ + import json + + listing = [_full("a.parquet")] + _install_fs(monkeypatch, listing) + _install_fake_parquets_query( + monkeypatch, + lambda urls, q: pd.DataFrame( + [ + {"flow": 0.5, "nudge": float("nan")}, + {"flow": 0.6, "nudge": float("nan")}, + {"flow": 0.7, "nudge": 0.01}, + ] + ), + ) + + result = logic.query_files_by_selector( + **SELECTOR, + query="SELECT * FROM output", + ) + + assert result.get("ok") is True + # Strict round-trip — emits with allow_nan=False so any residual NaN + # surfaces as a ValueError rather than silently producing invalid JSON. + round_tripped = json.loads(json.dumps(result, allow_nan=False)) + rows = round_tripped["data"] + assert rows[0]["nudge"] is None + assert rows[1]["nudge"] is None + assert rows[2]["nudge"] == 0.01 + assert rows[0]["flow"] == 0.5 + + # --------------------------------------------------------------------------- # XOR + Pydantic edge cases # ---------------------------------------------------------------------------