Score translations on the reactions they return

ord_schema/dependencies_test.py

@@ -43,7 +43,7 @@
     # The search subpackage reads through the artifacts, whose imports the search extra
     # therefore has to carry.
     "search": ("ord_schema.search", "ord_schema.artifacts"),
-    "nl": ("ord_schema.search.nl",),
+    "nl": ("ord_schema.search.nl", "ord_schema.search.nl_eval"),
     "orm": ("ord_schema.orm",),
     "huggingface": ("ord_schema.huggingface",),
 }

ord_schema/search/README.md

@@ -454,10 +454,36 @@ rather than guaranteed: the predicate tree usually arrives JSON-encoded in a str
 coerced back, a query that does not compile is handed back once carrying the compiler's
 own "did you mean", and a second failure raises `MalformedQueryError`.
 
+The model is also given a way to decline. Forcing `build_query` would leave it no way to
+say a question cannot be put to this grammar — comparing two columns, say — and a model
+with no way to decline invents a query rather than refusing, which is the failure that
+looks most like an answer. Declining raises `UnanswerableError`, carrying the model's
+reason, and is never repaired: nothing was wrong with its reasoning.
+
 The ~15k-token prefix — these rules plus `describe()` plus the grammar — is cached, which
 is most of what a query costs. `answer.query` is the query that ran, so a caller can show
 what was searched and offer to run it again.
 
+### Measure how good a translation is
+
+```bash
+python -m ord_schema.search.nl_eval \
+    --projections 'projections/**/*.parquet' \
+    --structures 'structures/**/*.parquet' \
+    --model claude-haiku-4-5
+```
+
+A case states **reactions any correct query returns** and reactions a plausible wrong one
+returns, never the query it expects: several spellings of a question are right, and pinning
+one would fail a better translation than the one written the day the case was added. The
+`must_not_return` half is what gives a case teeth — for "pyridine as a solvent" it holds
+reactions where pyridine is a reactant and something else is the solvent, which is what
+comes back when two conditions on one component become two quantifiers.
+
+Cases carry a `why`, printed with any failure, and one is marked `compiles: false`: a
+question the grammar cannot express, which the layer has to refuse rather than answer
+approximately. The reaction IDs come from the corpus the cases were built against.
+
 ### Tell the model what it may query
 
 ```python

ord_schema/search/nl.py

@@ -73,6 +73,25 @@
     "description": "Build an ORD search query from the user's question.",
     "input_schema": query.Query.model_json_schema(),
 }
+# Forcing build_query would leave a model with no way to decline, and a model with no
+# way to decline invents a query rather than refusing. This is that way.
+REFUSAL_TOOL: ToolParam = {
+    "name": "cannot_answer",
+    "description": (
+        "Say that the question cannot be expressed in this grammar. Use it rather than "
+        "building a query that means something else."
+    ),
+    "input_schema": {
+        "type": "object",
+        "properties": {
+            "reason": {
+                "type": "string",
+                "description": "What the question asks for that the grammar lacks.",
+            }
+        },
+        "required": ["reason"],
+    },
+}
 _SYSTEM: list[TextBlockParam] = [
     {
         "type": "text",
@@ -102,6 +121,16 @@ class MalformedQueryError(NLQueryError):
     """The model's query did not compile, and neither did its repair."""
 
 
+class UnanswerableError(NLQueryError):
+    """The question cannot be put to this grammar, and the model said so.
+
+    Distinct from a malformed query: nothing was wrong with the model's reasoning, and
+    retrying will not help. Comparing two columns is the standard case -- a value is a
+    literal or a compound, never another column -- and a layer without a way to say so
+    answers with a plausible query that means something else.
+    """
+
+
 @dataclasses.dataclass(frozen=True)
 class Answer:
     """What a question produced, including the query it became.
@@ -193,15 +222,22 @@ def _call(
             max_tokens=MAX_TOKENS,
             system=_SYSTEM,
             messages=messages,
-            tools=[TOOL],
-            tool_choice={"type": "tool", "name": "build_query"},
+            tools=[TOOL, REFUSAL_TOOL],
+            # "any" rather than "tool": the model must call one of them, which leaves
+            # refusing available without leaving prose available.
+            tool_choice={"type": "any"},
         )
     except anthropic.RateLimitError as error:
         raise ModelRateLimitedError(str(error)) from error
     except (anthropic.APIConnectionError, anthropic.APIStatusError) as error:
         raise ModelUnavailableError(str(error)) from error
     for block in response.content:
         if isinstance(block, ToolUseBlock):
+            if block.name == REFUSAL_TOOL["name"]:
+                reason = "no reason given"
+                if isinstance(block.input, dict):
+                    reason = str(block.input.get("reason", reason))
+                raise UnanswerableError(reason)
             return block
     raise MalformedQueryError("the model returned no query")
 
@@ -228,6 +264,7 @@ def translate(
     Raises:
         MalformedQueryError: If the query does not compile, after the repair turn where
             one was allowed.
+        UnanswerableError: If the model says the grammar cannot express the question.
         ModelRateLimitedError: If the caller is over its rate limit.
         ModelUnavailableError: If the model cannot be reached.
     """
@@ -363,6 +400,7 @@ def ask(
 
     Raises:
         MalformedQueryError: If translation produces nothing that compiles.
+        UnanswerableError: If the grammar cannot express the question.
         ModelRateLimitedError: If the caller is over its rate limit.
         ModelUnavailableError: If the model cannot be reached.
     """

ord_schema/search/nl_cases.json

@@ -0,0 +1,55 @@
+[
+  {
+    "question": "which reactions use pyridine as a solvent?",
+    "why": "two conditions on one element, which a wrong translation splits in two",
+    "must_return": [
+      "ord-00533d2621284180b8d6e372c9496cb7",
+      "ord-02572d7994bc44c39cdf4b16503d331d",
+      "ord-03179abb6e0a461e9c57bc862974a3e0"
+    ],
+    "must_not_return": [
+      "ord-9c0049ffd5bd4c2597bd8dc1da4f9125",
+      "ord-c218088b16c541088a933a7606f68c4c"
+    ]
+  },
+  {
+    "question": "reactions run above 350 K",
+    "why": "a scalar comparison needing no quantifier at all",
+    "must_return": [
+      "ord-01e7218d0d9042f6bb980d3c55b949be",
+      "ord-039db69849d0448baa4933c8edcbfd3d",
+      "ord-052ce82d4b04486e8815d005d6897bba"
+    ],
+    "must_not_return": []
+  },
+  {
+    "question": "reactions where a desired product has a yield above 50%",
+    "why": "correlation: the yield has to belong to the desired product, not to whichever product happens to carry one",
+    "must_return": [
+      "ord-004a7e7ef4f248aea8ceb23af8212300",
+      "ord-01e04eeb2be04ecb80d130e0c31917fe",
+      "ord-02e987dd13d54efaac154a47fe44f7e2"
+    ],
+    "must_not_return": [
+      "ord-038119b8b4624c1a8ebfcc86cea84ff6",
+      "ord-c867f2f5ee844d8099373f8a271335f5"
+    ]
+  },
+  {
+    "question": "reactions with no solvent at all",
+    "why": "a forall, which the occurrence index cannot answer",
+    "must_return": [
+      "ord-005bd91bc77a4958a35aa743cdeadabd",
+      "ord-01123dbb1eea4f0ca582cd558fc37de7",
+      "ord-0197aeb8022e45b88d3d52d2b1539148"
+    ],
+    "must_not_return": []
+  },
+  {
+    "question": "reactions that ran longer than their workup took",
+    "why": "comparing two columns, which the grammar cannot express: a value is a literal or a compound, never another column",
+    "compiles": false,
+    "must_return": [],
+    "must_not_return": []
+  }
+]