Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ Image tags follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Fixed

- **Stripped "NetCDF" tokens from tool descriptions; added "Parquet only" anchor to `list_available_output_files`.** The descriptions for `query_files_by_selector` (`_tool_descriptions.py`), its `index` arg (`tools.py:444`), and `list_available_output_files` (`tools.py:296`) had two related leaks: (1) `query_files_by_selector`'s description mentioned "NetCDF" four times in negative-constraint framing (naming `unsupported_format`, `_excluded_netcdf_count`, and the mixed-format selector behavior), with one more mention in the `index` Field description, and (2) `list_available_output_files`'s description had no parquet-only anchor at all — its name reads as generic "output files." Weak quantized models (observed against `SimonPu/GLM-4.7-Flash:Q4_K_M`) token-match on the format name regardless of polarity, so "NetCDF unsupported" reads as "supports NetCDF" to a Q4 4.7B-class model. The model then assigned the NetCDF role to the unanchored `list_available_output_files` tool and emitted user-facing replies routing NetCDF prompts to it — directly contradicting the v0.5.0 parquet-only decision. Tightened all three description sites: hoist "Parquet only." up next to the opener of `query_files_by_selector`; append "Parquet only." to `list_available_output_files`; drop the "NetCDF files do not consume index slots" sentence from the `index` Field description; drop all "netcdf" / `unsupported_format` / `_excluded_netcdf_count` mentions from description prose. The runtime envelope keys keep working — we just stop teaching the LLM their names from the system prompt. `test_tool_descriptions.py` updated: drop the positive `unsupported_format` / `_excluded_netcdf_count` assertions; add a negative `"netcdf" not in desc.lower()` assertion plus a positive `"parquet only"` assertion for `list_available_output_files`.

## [0.5.2] - 2026-05-21

### Changed
Expand Down
26 changes: 16 additions & 10 deletions nextgen_mcp/_tool_descriptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,31 @@
can lock the content against drift via positive + negative substring assertions.

Description content must obey:
- Positive: name the load-bearing constraints (parquet-only, the error
classes the LLM may receive, provenance columns).
- Positive: name the load-bearing constraints (parquet-only, provenance
columns, filter arg shape).
- Negative: no concrete example values. LLMs copy concrete examples
verbatim into tool calls. No ``s3://`` URLs, no example filenames,
no inline SQL.
no inline SQL. Also no negative-form mentions of formats we don't
support — weak quantized models token-match on the format name
regardless of polarity, so naming "NetCDF" even as an unsupported
case leaks the token into the model's working set.
"""

QUERY_FILES_BY_SELECTOR_DESCRIPTION = (
"Query NRDS parquet outputs by selector. Queries one OR many files "
"in a single call: pass `file_name` or `index` to filter to one file; "
"omit both to query the full selector as a unioned dataset. "
"Parquet only - NetCDF files return an `unsupported_format:` envelope. "
"Query NRDS parquet outputs by selector. Parquet only. "
"Queries one OR many files in a single call: pass `file_name` or "
"`index` to filter to one file; omit both to query the full selector "
"as a unioned dataset. "
"Result rows always carry `filename` and `source_path` provenance "
"columns so SQL can group or filter by source. "
"Mixed-format selectors (parquet plus NetCDF) silently filter to "
"parquet and surface the exclusion count as `_excluded_netcdf_count` "
"on the result envelope when non-zero. "
"The SQL must be a single read-only SELECT or WITH...SELECT against "
"table `output`. For data extraction, prefer WHERE filtering over "
"LIMIT - LIMIT silently drops rows and breaks ordered time series. "
"Use aggregates (COUNT, SUM, AVG, MAX, MIN) for summary statistics."
)

LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION = (
"List available output files for a given model, date, forecast, cycle, "
"and VPU (accepts id or label, including subregion VPUs). Optional "
"ensemble member for applicable forecast. Parquet only."
)
13 changes: 8 additions & 5 deletions nextgen_mcp/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
lookup_hydrofabric_feature as _lookup_hydrofabric_feature,
get_hydrofabric_pmtiles_layers
)
from ._tool_descriptions import QUERY_FILES_BY_SELECTOR_DESCRIPTION
from ._tool_descriptions import (
LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION,
QUERY_FILES_BY_SELECTOR_DESCRIPTION,
)

from .middleware._input_validation_middleware import InvalidLLMInputError

Expand Down Expand Up @@ -293,7 +296,7 @@ def list_available_vpus_tool(

@mcp.tool(
name="list_available_output_files",
description="List available output files for a given model, date, forecast, cycle, and VPU (accepts id or label, including subregion VPUs). Optional ensemble member for applicable forecast.",
description=LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION,
)
def list_available_output_files_tool(
model: Annotated[MODELS, Field(description="Model id - call list_available_models to discover valid values")] = None,
Expand Down Expand Up @@ -437,9 +440,9 @@ def query_files_by_selector_tool(
Optional[int],
Field(
description=(
"Optional filter to one file by 0-based index into the parquet-only "
"sorted file list. NetCDF files do not consume index slots. Mutually "
"exclusive with file_name."
"Optional filter to one file by 0-based index into the sorted "
"parquet file list for the selector. Mutually exclusive with "
"file_name."
),
ge=0,
),
Expand Down
33 changes: 26 additions & 7 deletions test_mcp/test_tool_descriptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@

import re

from nextgen_mcp._tool_descriptions import QUERY_FILES_BY_SELECTOR_DESCRIPTION
from nextgen_mcp._tool_descriptions import (
LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION,
QUERY_FILES_BY_SELECTOR_DESCRIPTION,
)


def test_query_files_by_selector_description_positive_invariants() -> None:
Expand All @@ -22,9 +25,6 @@ def test_query_files_by_selector_description_positive_invariants() -> None:
# Names the parquet-only constraint
assert "parquet only" in lower

# Names the error class the LLM may receive on NetCDF-target calls
assert "unsupported_format" in desc

# Names the provenance columns the LLM can reference in SQL
assert "filename" in desc
assert "source_path" in desc
Expand All @@ -33,9 +33,6 @@ def test_query_files_by_selector_description_positive_invariants() -> None:
assert "file_name" in desc
assert "index" in desc

# Names _excluded_netcdf_count so the LLM knows to look for it
assert "_excluded_netcdf_count" in desc


def test_query_files_by_selector_description_negative_invariants() -> None:
"""Description must not embed concrete example values that LLMs would copy."""
Expand Down Expand Up @@ -64,3 +61,25 @@ def test_query_files_by_selector_description_negative_invariants() -> None:
assert "cfe_nom" not in desc
assert "short_range" not in desc
assert not re.search(r"\bVPU_\d+", desc)

# No "netcdf" mentions — weak quantized models token-match on the word
# regardless of polarity ("NetCDF unsupported" reads as "supports NetCDF"
# to a Q4 4.7B-class model). Anchor positive-only ("Parquet only").
# Runtime envelope keys like `_excluded_netcdf_count` keep working; we
# just stop teaching the LLM their name from the description.
assert "netcdf" not in desc.lower()
assert "unsupported_format" not in desc


def test_list_available_output_files_description_anchor() -> None:
"""list_available_output_files description must carry a 'parquet only'
anchor and contain no 'netcdf' tokens — same reasoning as the
query_files_by_selector negative invariants. Without an explicit
parquet anchor on a generically named 'output files' tool, weak
quantized models route NetCDF-shaped prompts here.
"""
desc = LIST_AVAILABLE_OUTPUT_FILES_DESCRIPTION
lower = desc.lower()

assert "parquet only" in lower
assert "netcdf" not in lower
Loading