Make the buddy the tutor along the onboarding path

src/api/routes/grading.py

@@ -20,6 +20,8 @@
 
 router = APIRouter()
 
+_MAX_GRADING_ATTEMPTS = 2
+
 SYSTEM_PROMPT = """
 You grade short-text answers to onboarding knowledge-check questions.
 
@@ -47,7 +49,7 @@ class _GradedItem(BaseModel):
 
 
 class _Payload(BaseModel):
-    results: list[_GradedItem] = []
+    results: list[_GradedItem]
 
 
 def _build_prompt(answers: list[GradeAnswerItem]) -> list[Message]:
@@ -64,6 +66,62 @@ def _build_prompt(answers: list[GradeAnswerItem]) -> list[Message]:
     ]
 
 
+def _grade(llm: LLMClient, to_grade: list[GradeAnswerItem]) -> dict[str, _GradedItem]:
+    """Grade in one call, asking once more when the reply is unusable.
+
+    An unreadable reply marks every answer in it "could not be graded" -- which the
+    caller records as wrong. A small local model breaks its JSON now and then, and
+    a hire whose right answer came back wrong for that reason has been told
+    something false about their own knowledge. One correction round is cheap
+    against that.
+
+    Valid JSON is not enough: a reply that leaves an answer out (``{}``, or one id
+    missing) would record that answer as wrong just the same, so it gets the same
+    correction round. It must hold exactly one grade per requested id.
+    """
+    expected_ids = {item.id for item in to_grade}
+    messages = _build_prompt(to_grade)
+    for attempt in range(_MAX_GRADING_ATTEMPTS):
+        try:
+            raw = llm.generate(messages)
+        except LLMUnavailableError as exc:
+            raise HTTPException(status_code=503, detail=str(exc)) from exc
+        try:
+            payload = _Payload.model_validate_json(extract_json_object(raw))
+            graded_by_id = {item.id: item for item in payload.results}
+            if (
+                len(payload.results) != len(to_grade)
+                or set(graded_by_id) != expected_ids
+            ):
+                raise ValueError(
+                    "grading response does not match the requested answer ids: "
+                    f"expected {sorted(expected_ids)}, got "
+                    f"{[item.id for item in payload.results]}"
+                )
+            return graded_by_id
+        except (ValidationError, json.JSONDecodeError, ValueError) as exc:
+            logger.warning(
+                "Could not parse grade-answers output (attempt %d): %s",
+                attempt + 1,
+                exc,
+            )
+            if attempt + 1 == _MAX_GRADING_ATTEMPTS:
+                break
+            messages = [
+                *messages,
+                Message(role="assistant", content=raw),
+                Message(
+                    role="user",
+                    content=(
+                        f"That response could not be validated: {exc}. Return the same "
+                        "grades again as one valid JSON object matching the schema "
+                        "exactly. Return JSON only."
+                    ),
+                ),
+            ]
+    return {}
+
+
 @router.post(
     "/grade-answers",
     summary="Semantically grade short-text knowledge-check answers",
@@ -121,17 +179,7 @@ def grade_answers(
             )
 
     if to_grade:
-        try:
-            raw = llm.generate(_build_prompt(to_grade))
-        except LLMUnavailableError as exc:
-            raise HTTPException(status_code=503, detail=str(exc)) from exc
-
-        try:
-            payload = _Payload.model_validate_json(extract_json_object(raw))
-            graded_by_id = {item.id: item for item in payload.results}
-        except (ValidationError, json.JSONDecodeError, ValueError) as exc:
-            logger.warning("Could not parse grade-answers output: %s", exc)
-            graded_by_id = {}
+        graded_by_id = _grade(llm, to_grade)
 
         for item in to_grade:
             graded = graded_by_id.get(item.id)

src/onboarding/buddy_agent.py

@@ -68,7 +68,28 @@
 _SOURCE_CHARS = 800
 # How many internal search hops before we force a final answer, so a confused model
 # can't loop forever gathering evidence it never uses.
-_MAX_STEPS = 4
+#
+# Raised from 4 after a testing session: only *search-only* hops consume this budget (a
+# hop that asks for a backend tool returns immediately), and a model that searched four
+# times before deciding to act never got to act at all -- it hit the forced answer
+# below, which has no tools, and promised the hire a button that could not exist. Six
+# leaves room for a thorough answer and still bounds the loop.
+_MAX_STEPS = 6
+
+# What the model is told when the search budget is spent.
+#
+# It answers the hire's question with the persona still in front of it -- "offer
+# `add_path_step`", "offer to claim it" -- and without this it took those at face value
+# and told the hire to confirm something no tool call had produced. A model that knows
+# it has no tools this turn can only do the honest thing: answer with what it has and
+# leave the offer for the next turn.
+_NO_TOOLS_THIS_TURN = (
+    "You have no tools available for this reply and cannot do anything or offer "
+    "anything: no confirm button can appear. Answer with what you already have. Do not "
+    "say you have done something, do not tell the hire to confirm, click or check "
+    "anything, and do not claim anything is on their screen. If something still needs "
+    "doing, say what it is and that you can set it up when they reply."
+)
 
 
 @dataclass
@@ -316,7 +337,11 @@ def run_agent_turn(
         # Only local searches this turn -- loop and let the model reason over them.
 
     # Step budget spent: force a final answer with no tools rather than loop forever.
-    forced = llm.generate(work)
+    #
+    # The notice goes to the model, not into the returned conversation: a final turn's
+    # messages are discarded by the caller, and a transcript carrying instructions about
+    # a budget nobody can see would be a strange thing to keep.
+    forced = llm.generate([*work, Message(role="system", content=_NO_TOOLS_THIS_TURN)])
     return AgentTurnResult(
         final=True,
         text=forced,