diff --git a/CHANGELOG.md b/CHANGELOG.md index 75c5c0a..193b601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ Image tags follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed + +- **Stripped "NetCDF" tokens from tool descriptions; added "Parquet only" anchor to `list_available_output_files`.** The descriptions for `query_files_by_selector` (`_tool_descriptions.py`), its `index` arg (`tools.py:444`), and `list_available_output_files` (`tools.py:296`) had two related leaks: (1) `query_files_by_selector`'s description mentioned "NetCDF" four times in negative-constraint framing (naming `unsupported_format`, `_excluded_netcdf_count`, and the mixed-format selector behavior), with one more mention in the `index` Field description, and (2) `list_available_output_files`'s description had no parquet-only anchor at all — its name reads as generic "output files." Weak quantized models (observed against `SimonPu/GLM-4.7-Flash:Q4_K_M`) token-match on the format name regardless of polarity, so "NetCDF unsupported" reads as "supports NetCDF" to a Q4 4.7B-class model. The model then assigned the NetCDF role to the unanchored `list_available_output_files` tool and emitted user-facing replies routing NetCDF prompts to it — directly contradicting the v0.5.0 parquet-only decision. Tightened all three description sites: hoist "Parquet only." up next to the opener of `query_files_by_selector`; append "Parquet only." to `list_available_output_files`; drop the "NetCDF files do not consume index slots" sentence from the `index` Field description; drop all "netcdf" / `unsupported_format` / `_excluded_netcdf_count` mentions from description prose. The runtime envelope keys keep working — we just stop teaching the LLM their names from the system prompt. `test_tool_descriptions.py` updated: drop the positive `unsupported_format` / `_excluded_netcdf_count` assertions; add a negative `"netcdf" not in desc.lower()` assertion plus a positive `"parquet only"` assertion for `list_available_output_files`. + ## [0.5.2] - 2026-05-21 ### Changed diff --git a/nextgen_mcp/_tool_descriptions.py b/nextgen_mcp/_tool_descriptions.py index 91fd511..5b986c9 100644 --- a/nextgen_mcp/_tool_descriptions.py +++ b/nextgen_mcp/_tool_descriptions.py @@ -4,25 +4,31 @@ can lock the content against drift via positive + negative substring assertions. Description content must obey: - - Positive: name the load-bearing constraints (parquet-only, the error - classes the LLM may receive, provenance columns). + - Positive: name the load-bearing constraints (parquet-only, provenance + columns, filter arg shape). - Negative: no concrete example values. LLMs copy concrete examples verbatim into tool calls. No ``s3://`` URLs, no example filenames, - no inline SQL. + no inline SQL. Also no negative-form mentions of formats we don't + support — weak quantized models token-match on the format name + regardless of polarity, so naming "NetCDF" even as an unsupported + case leaks the token into the model's working set. """ QUERY_FILES_BY_SELECTOR_DESCRIPTION = ( - "Query NRDS parquet outputs by selector. Queries one OR many files " - "in a single call: pass `file_name` or `index` to filter to one file; " - "omit both to query the full selector as a unioned dataset. " - "Parquet only - NetCDF files return an `unsupported_format:` envelope. " + "Query NRDS parquet outputs by selector. Parquet only. " + "Queries one OR many files in a single call: pass `file_name` or " + "`index` to filter to one file; omit both to query the full selector " + "as a unioned dataset. " "Result rows always carry `filename` and `source_path` provenance " "columns so SQL can group or filter by source. " - "Mixed-format selectors (parquet plus NetCDF) silently filter to " - "parquet and surface the exclusion count as `_excluded_netcdf_count` " - "on the result envelope when non-zero. " "The SQL must be a single read-only SELECT or WITH...SELECT against " "table `output`. For data extraction, prefer WHERE filtering over " "LIMIT - LIMIT silently drops rows and breaks ordered time series. " "Use aggregates (COUNT, SUM, AVG, MAX, MIN) for summary statistics." ) + +LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION = ( + "List available output files for a given model, date, forecast, cycle, " + "and VPU (accepts id or label, including subregion VPUs). Optional " + "ensemble member for applicable forecast. Parquet only." +) diff --git a/nextgen_mcp/tools.py b/nextgen_mcp/tools.py index 7a9ddec..1f6eb04 100644 --- a/nextgen_mcp/tools.py +++ b/nextgen_mcp/tools.py @@ -34,7 +34,10 @@ lookup_hydrofabric_feature as _lookup_hydrofabric_feature, get_hydrofabric_pmtiles_layers ) -from ._tool_descriptions import QUERY_FILES_BY_SELECTOR_DESCRIPTION +from ._tool_descriptions import ( + LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION, + QUERY_FILES_BY_SELECTOR_DESCRIPTION, +) from .middleware._input_validation_middleware import InvalidLLMInputError @@ -293,7 +296,7 @@ def list_available_vpus_tool( @mcp.tool( name="list_available_output_files", - description="List available output files for a given model, date, forecast, cycle, and VPU (accepts id or label, including subregion VPUs). Optional ensemble member for applicable forecast.", + description=LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION, ) def list_available_output_files_tool( model: Annotated[MODELS, Field(description="Model id - call list_available_models to discover valid values")] = None, @@ -437,9 +440,9 @@ def query_files_by_selector_tool( Optional[int], Field( description=( - "Optional filter to one file by 0-based index into the parquet-only " - "sorted file list. NetCDF files do not consume index slots. Mutually " - "exclusive with file_name." + "Optional filter to one file by 0-based index into the sorted " + "parquet file list for the selector. Mutually exclusive with " + "file_name." ), ge=0, ), diff --git a/test_mcp/test_tool_descriptions.py b/test_mcp/test_tool_descriptions.py index f6e0aaf..cba9467 100644 --- a/test_mcp/test_tool_descriptions.py +++ b/test_mcp/test_tool_descriptions.py @@ -12,7 +12,10 @@ import re -from nextgen_mcp._tool_descriptions import QUERY_FILES_BY_SELECTOR_DESCRIPTION +from nextgen_mcp._tool_descriptions import ( + LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION, + QUERY_FILES_BY_SELECTOR_DESCRIPTION, +) def test_query_files_by_selector_description_positive_invariants() -> None: @@ -22,9 +25,6 @@ def test_query_files_by_selector_description_positive_invariants() -> None: # Names the parquet-only constraint assert "parquet only" in lower - # Names the error class the LLM may receive on NetCDF-target calls - assert "unsupported_format" in desc - # Names the provenance columns the LLM can reference in SQL assert "filename" in desc assert "source_path" in desc @@ -33,9 +33,6 @@ def test_query_files_by_selector_description_positive_invariants() -> None: assert "file_name" in desc assert "index" in desc - # Names _excluded_netcdf_count so the LLM knows to look for it - assert "_excluded_netcdf_count" in desc - def test_query_files_by_selector_description_negative_invariants() -> None: """Description must not embed concrete example values that LLMs would copy.""" @@ -64,3 +61,25 @@ def test_query_files_by_selector_description_negative_invariants() -> None: assert "cfe_nom" not in desc assert "short_range" not in desc assert not re.search(r"\bVPU_\d+", desc) + + # No "netcdf" mentions — weak quantized models token-match on the word + # regardless of polarity ("NetCDF unsupported" reads as "supports NetCDF" + # to a Q4 4.7B-class model). Anchor positive-only ("Parquet only"). + # Runtime envelope keys like `_excluded_netcdf_count` keep working; we + # just stop teaching the LLM their name from the description. + assert "netcdf" not in desc.lower() + assert "unsupported_format" not in desc + + +def test_list_available_output_files_description_anchor() -> None: + """list_available_output_files description must carry a 'parquet only' + anchor and contain no 'netcdf' tokens — same reasoning as the + query_files_by_selector negative invariants. Without an explicit + parquet anchor on a generically named 'output files' tool, weak + quantized models route NetCDF-shaped prompts here. + """ + desc = LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION + lower = desc.lower() + + assert "parquet only" in lower + assert "netcdf" not in lower