feat(quiz): confidence rating — pre-reveal prompt, calibration, and agent visibility

src/cognit/engine/models.py

@@ -108,6 +108,7 @@ class QuestionResult(BaseModel):
     correct: bool
     score: int = Field(ge=0, le=100)  # 0..100
     feedback: str  # "" for deterministic questions
+    confidence: int | None = Field(default=None, ge=1, le=5)  # reader's 1–5 self-rating, if given
 
 
 class Results(BaseModel):

src/cognit/mcp/assets/quiz_mcp.js

@@ -122,6 +122,16 @@ async function postAnswer(qid, value) {
   }
 }
 
+// Persist a confidence rating to the server (fire-and-forget) so the host agent can see
+// confidence vs. correctness. Local state is authoritative for the UI; this just mirrors it.
+function postConfidence(qid, value) {
+  fetch("/confidence", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ question_id: qid, value }),
+  }).catch((e) => console.error("confidence POST failed:", e));
+}
+
 // Re-POST every non-empty local answer (idempotent) so the server has everything
 // before grading — covers open answers that only POST on blur.
 async function flushAnswers() {
@@ -411,10 +421,59 @@ function isCorrectLocal(q) {
   return q.type === "tf" ? v === String(q.answer) : v === q.answer;
 }
 
-// Practice mode: a committed deterministic question reveals immediately and locks.
-function shouldReveal(q) { return !examMode && isDeterministic(q) && isAnswered(q); }
+// Practice mode, after committing a deterministic question: first rate confidence,
+// then reveal. Confidence is local + honor-system (the answer's already in the page).
+const confidence = {};  // qid -> 1..5
+const CONFIDENCE_SCALE = [
+  [1, "Guessing"], [2, "Unsure"], [3, "Maybe"], [4, "Fairly sure"], [5, "Certain"],
+];
+function needsConfidence(q) {
+  return !examMode && isDeterministic(q) && isAnswered(q) && confidence[q.id] == null;
+}
+function shouldReveal(q) {
+  return !examMode && isDeterministic(q) && isAnswered(q) && confidence[q.id] != null;
+}
+
+// Calibration verdict from (confidence, correctness): the teachable case is being
+// confident and wrong. Returns null when nothing notable.
+function calibration(q) {
+  const c = confidence[q.id];
+  if (c == null) return null;
+  const correct = isCorrectLocal(q);
+  if (!correct && c >= 4) return { cls: "over", text: "Confident but wrong — worth a closer look." };
+  if (correct && c <= 2) return { cls: "under", text: "Right — but you weren't sure. Lock in why." };
+  return null;
+}
+
+function renderConfidencePrompt(q, i) {
+  const pick = (n) => { confidence[q.id] = n; postConfidence(q.id, n); renderQuestions(); };
+  const scale = el("div", { class: "confidence__scale", role: "radiogroup", "aria-label": "Confidence" },
+    CONFIDENCE_SCALE.map(([n, label]) => el("button", {
+      class: "confidence__btn", type: "button", title: label, "aria-label": `${n} — ${label}`,
+      text: String(n), onclick: () => pick(n),
+    })));
+  return el("article", { class: "file" }, [
+    el("div", { class: "file__head" }, [
+      el("div", { class: "file__title", text: `Question ${i + 1}` }),
+      el("div", { class: "file__type", text: TYPE_LABEL[q.type] }),
+    ]),
+    el("div", { class: "file__body" }, [
+      el("p", { class: "prompt" }, renderPrompt(q.prompt)),
+      renderAnchor(q),
+      el("div", { class: "confidence" }, [
+        el("div", { class: "confidence__q", text: "How sure are you?" }),
+        scale,
+        el("div", { class: "confidence__ends" }, [
+          el("span", { text: "1 · guessing" }),
+          el("span", { text: "5 · certain" }),
+        ]),
+      ]),
+    ]),
+  ]);
+}
 
 function renderQuestion(q, i) {
+  if (needsConfidence(q)) return renderConfidencePrompt(q, i);  // rate confidence, then reveal
   if (shouldReveal(q)) {
     // Reuse the results card (correct/your-pick rows + explanation), with a locally
     // computed verdict — no server round-trip, the answer is already in /state.
@@ -653,6 +712,14 @@ function renderResultCard(q, r, i) {
       el("p", { text: q.explanation }),
     ]));
   }
+  if (confidence[q.id] != null) {
+    const cal = calibration(q);
+    const label = (CONFIDENCE_SCALE.find(([n]) => n === confidence[q.id]) || [, ""])[1];
+    body.push(el("div", { class: "calibration" + (cal ? ` calibration--${cal.cls}` : "") }, [
+      el("span", { class: "calibration__rating", text: `Confidence ${confidence[q.id]}/5 · ${label}` }),
+      cal ? el("span", { class: "calibration__flag", text: cal.text }) : null,
+    ]));
+  }
   return el("article", { class: `file ${cls}` }, [
     el("div", { class: "file__head" }, [
       el("div", { class: "file__title", text: `Question ${i + 1}` }),

src/cognit/mcp/assets/styles.css

@@ -885,3 +885,41 @@ a.codepanel__file:hover { color: var(--blue); text-decoration: underline; }
   overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
 }
 .sidelist.coverage li.uncovered .coverage__file { color: var(--fg-mute); }
+
+/* confidence rating (practice mode, before reveal) ───────────────── */
+.confidence {
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 16px;
+  background: var(--bg-subtle);
+}
+.confidence__q { font-size: 14px; font-weight: 600; color: var(--fg); margin-bottom: 12px; }
+.confidence__scale { display: flex; gap: 8px; }
+.confidence__btn {
+  flex: 1; min-width: 0;
+  padding: 12px 0;
+  font: 600 15px var(--mono);
+  color: var(--fg);
+  background: var(--bg-canvas);
+  border: 1px solid var(--border);
+  border-radius: 6px;
+  cursor: pointer;
+  transition: border-color 120ms, background 120ms;
+}
+.confidence__btn:hover { border-color: var(--blue); background: var(--blue-bg); }
+.confidence__btn:focus-visible { outline: 2px solid var(--blue); outline-offset: 2px; }
+.confidence__ends { display: flex; justify-content: space-between; margin-top: 6px; font-size: 11px; color: var(--fg-faint); }
+
+/* calibration verdict (shown on the revealed card) */
+.calibration {
+  display: flex; flex-wrap: wrap; align-items: baseline; gap: 6px 12px;
+  margin-top: 12px; padding: 8px 12px;
+  border-radius: 6px;
+  font-size: 13px;
+  background: var(--bg-subtle);
+  color: var(--fg-mute);
+}
+.calibration__rating { font-family: var(--mono); font-size: 12px; }
+.calibration__flag { font-weight: 500; }
+.calibration--over { background: var(--red-bg); color: var(--red); }
+.calibration--under { background: var(--orange-bg); color: var(--orange); }

src/cognit/mcp/grading.py

@@ -20,5 +20,17 @@ def grade_state(state: QuizState, *, llm: LLMClient) -> Results:
         entries=[AnswerEntry(question_id=qid, value=val) for qid, val in answers_map.items()],
     )
     results = grade(quiz, answers, llm=llm)
+    # Attach the reader's self-reported confidence (1–5) to each result so the agent can
+    # see confidence vs. correctness — e.g. to follow up on confident-but-wrong questions.
+    confidences = dict(state.confidences)
+    if confidences:
+        results = results.model_copy(
+            update={
+                "per_question": [
+                    r.model_copy(update={"confidence": confidences.get(r.question_id)})
+                    for r in results.per_question
+                ]
+            }
+        )
     state.set_results(results)
     return results

src/cognit/mcp/server.py

@@ -60,7 +60,7 @@ def do_replace_question(state: QuizState, index: int, question: dict[str, Any])
 
 def do_get_answers(state: QuizState) -> dict[str, Any]:
     snap = state.snapshot()
-    return {"answers": snap["answers"], "quiz": snap["quiz"]}
+    return {"answers": snap["answers"], "confidences": snap["confidences"], "quiz": snap["quiz"]}
 
 
 def do_grade(state: QuizState, *, llm: LLMClient) -> dict[str, Any]:
@@ -137,7 +137,10 @@ def replace_question(index: int, question: dict[str, Any]) -> dict[str, Any]:
 
     @mcp.tool()
     def get_answers() -> dict[str, Any]:
-        """Read back the answers the developer selected in the browser + the current quiz."""
+        """Read back what the developer did in the browser: their `answers`, their
+        per-question `confidences` (1–5 self-ratings, where given), and the current quiz.
+        Confidence vs. correctness is useful for follow-ups — e.g. probe again where they
+        were confident but wrong, or drill a concept they were unsure about."""
         return do_get_answers(state)
 
     @mcp.tool()

src/cognit/mcp/state.py

@@ -21,6 +21,7 @@ def __init__(self, *, pr_number: int, snapshot_path: Path) -> None:
         self._lock = threading.Lock()
         self.quiz: Quiz | None = None
         self.answers: dict[str, str] = {}
+        self.confidences: dict[str, int] = {}  # question_id -> reader's 1–5 self-rating
         self.results: Results | None = None
         self._load()
 
@@ -34,13 +35,15 @@ def _load(self) -> None:
         if data.get("quiz"):
             self.quiz = Quiz.model_validate(data["quiz"])
         self.answers = dict(data.get("answers") or {})
+        self.confidences = {k: int(v) for k, v in (data.get("confidences") or {}).items()}
         if data.get("results"):
             self.results = Results.model_validate(data["results"])
 
     def _persist(self) -> None:
         payload = {
             "quiz": self.quiz.model_dump(mode="json") if self.quiz else None,
             "answers": self.answers,
+            "confidences": self.confidences,
             "results": self.results.model_dump(mode="json") if self.results else None,
         }
         self._snapshot_path.write_text(json.dumps(payload))
@@ -49,6 +52,7 @@ def set_quiz(self, quiz: Quiz) -> None:
         with self._lock:
             self.quiz = quiz
             self.answers = {}
+            self.confidences = {}
             self.results = None
             self._persist()
 
@@ -61,6 +65,7 @@ def replace_question(self, index: int, question: Question) -> None:
             qs[index] = question
             self.quiz = self.quiz.model_copy(update={"questions": qs})
             self.answers.pop(old_id, None)
+            self.confidences.pop(old_id, None)
             self.results = None
             self._persist()
 
@@ -69,6 +74,11 @@ def record_answer(self, question_id: str, value: str) -> None:
             self.answers[question_id] = value
             self._persist()
 
+    def record_confidence(self, question_id: str, value: int) -> None:
+        with self._lock:
+            self.confidences[question_id] = value
+            self._persist()
+
     def set_results(self, results: Results) -> None:
         with self._lock:
             self.results = results
@@ -95,5 +105,6 @@ def snapshot(self) -> dict[str, object]:
             return {
                 "quiz": self.quiz.model_dump(mode="json") if self.quiz else None,
                 "answers": dict(self.answers),
+                "confidences": dict(self.confidences),
                 "results": self.results.model_dump(mode="json") if self.results else None,
             }

src/cognit/mcp/web.py

@@ -1,14 +1,17 @@
 """FastAPI app: the browser projection over QuizState.
 
 Endpoints:
-  GET  /state    — JSON {quiz, answers, results}; the browser polls this
-  POST /answer   — {question_id, value} → record a browser-side answer
-  POST /grade    — human-triggered "Submit quiz": grade now (handler-owned, same path
-                   the agent's `grade` tool uses) and store results. Returns the Results.
-  POST /publish  — human-gated: render + post the results scorecard comment (reuses
-                   ghio.pr.post_comment). The ONLY outward-facing action; never an agent tool.
-  GET  /         — the quiz page (polls /state)
-  GET  /static/* — bundled assets
+  GET  /state         — JSON {quiz, answers, confidences, results}; the browser polls this
+  POST /answer        — {question_id, value} → record a browser-side answer
+  POST /confidence    — {question_id, value:1-5} → record the reader's confidence rating
+  GET  /diff          — ?path= → the unified-diff section for one changed file (inline hunks)
+  GET  /changed-files — JSON {files:[...]} for the diff coverage map
+  POST /grade         — human-triggered "Submit quiz": grade now (handler-owned, same path
+                        the agent's `grade` tool uses) and store results. Returns the Results.
+  POST /publish       — human-gated: render + post the results scorecard comment (reuses
+                        ghio.pr.post_comment). The ONLY outward-facing action; never an agent tool.
+  GET  /              — the quiz page (polls /state)
+  GET  /static/*      — bundled assets
 """
 
 from __future__ import annotations
@@ -82,6 +85,21 @@ async def post_answer(req: Request) -> JSONResponse:
         state.record_answer(qid, value)
         return JSONResponse({"ok": True})
 
+    @app.post("/confidence")
+    async def post_confidence(req: Request) -> JSONResponse:
+        body = await req.json()
+        qid, value = body.get("question_id"), body.get("value")
+        # bool is an int subclass — reject it explicitly so True/False can't sneak through.
+        if not isinstance(qid, str) or isinstance(value, bool) or not isinstance(value, int):
+            return JSONResponse(
+                {"ok": False, "error": "question_id (string) and value (int) required"},
+                status_code=422,
+            )
+        if not (1 <= value <= 5):
+            return JSONResponse({"ok": False, "error": "value must be 1–5"}, status_code=422)
+        state.record_confidence(qid, value)
+        return JSONResponse({"ok": True})
+
     @app.get("/diff", response_class=PlainTextResponse)
     def get_diff(path: str = "") -> PlainTextResponse:
         if diff_section is None:

tests/engine/test_models.py

@@ -111,6 +111,16 @@ def test_objective_questions_carry_optional_explanation():
     )
 
 
+def test_question_result_confidence_optional():
+    r = QuestionResult(question_id="q1", correct=True, score=100, feedback="")
+    assert r.confidence is None  # backward compatible: defaults to None
+    r2 = QuestionResult(question_id="q1", correct=False, score=0, feedback="", confidence=5)
+    assert r2.confidence == 5
+    for bad in (0, 6):
+        with pytest.raises(ValidationError):
+            QuestionResult(question_id="q1", correct=True, score=100, feedback="", confidence=bad)
+
+
 def test_anchor_round_trip():
     a = Anchor(path="src/cognit/mcp/state.py", start_line=40, end_line=46)
     assert Anchor.model_validate(a.model_dump()) == a

tests/mcp/test_state.py

@@ -46,6 +46,39 @@ def test_replace_question_drops_old_answer(tmp_path: Path) -> None:
     assert s.quiz is not None and s.quiz.questions[0].id == "q1b"  # new question in place
 
 
+def test_record_confidence(tmp_path: Path) -> None:
+    s = QuizState(pr_number=7, snapshot_path=tmp_path / "s.json")
+    s.set_quiz(_quiz())
+    s.record_confidence("q1", 4)
+    assert s.confidences == {"q1": 4}
+
+
+def test_confidences_persisted_and_reloaded(tmp_path: Path) -> None:
+    snap = tmp_path / "s.json"
+    s = QuizState(pr_number=7, snapshot_path=snap)
+    s.set_quiz(_quiz())
+    s.record_confidence("q1", 3)
+    s2 = QuizState(pr_number=7, snapshot_path=snap)
+    assert s2.confidences == {"q1": 3}
+    assert s2.snapshot()["confidences"] == {"q1": 3}
+
+
+def test_set_quiz_resets_confidences(tmp_path: Path) -> None:
+    s = QuizState(pr_number=7, snapshot_path=tmp_path / "s.json")
+    s.set_quiz(_quiz())
+    s.record_confidence("q1", 5)
+    s.set_quiz(_quiz())
+    assert s.confidences == {}
+
+
+def test_replace_question_drops_confidence(tmp_path: Path) -> None:
+    s = QuizState(pr_number=7, snapshot_path=tmp_path / "s.json")
+    s.set_quiz(_quiz())
+    s.record_confidence("q1", 5)
+    s.replace_question(0, MCQQuestion(id="q1b", prompt="p2", options=["X", "Y"], answer="Y"))
+    assert "q1" not in s.confidences
+
+
 def test_corrupt_snapshot_loads_as_empty(tmp_path: Path) -> None:
     snap = tmp_path / "s.json"
     snap.write_text("{not valid json")

tests/mcp/test_tools.py

@@ -88,6 +88,25 @@ def test_get_answers(tmp_path: Path):
     assert out["quiz"]["questions"][0]["id"] == "q1"
 
 
+def test_get_answers_includes_confidences(tmp_path: Path):
+    state = _state(tmp_path)
+    srv.do_set_quiz(state, _draft())
+    state.record_answer("q1", "A")
+    state.record_confidence("q1", 2)
+    out = srv.do_get_answers(state)
+    assert out["confidences"] == {"q1": 2}
+
+
+def test_grade_attaches_confidence(tmp_path: Path):
+    state = _state(tmp_path)
+    srv.do_set_quiz(state, _draft())
+    state.record_answer("q1", "A")
+    state.record_confidence("q1", 4)
+    out = srv.do_grade(state, llm=FakeLLM())
+    assert out["ok"] is True
+    assert out["per_question"][0]["confidence"] == 4
+
+
 def test_grade_without_quiz_returns_structured_failure(tmp_path: Path):
     out = srv.do_grade(_state(tmp_path), llm=FakeLLM())
     assert out["ok"] is False

tests/mcp/test_web.py

@@ -72,6 +72,19 @@ def test_post_answer_records(client):
     assert state.answers == {"q1": "A"}
 
 
+def test_post_confidence_records(client):
+    c, state, _ = client
+    assert c.post("/confidence", json={"question_id": "q1", "value": 3}).status_code == 200
+    assert state.confidences == {"q1": 3}
+
+
+def test_post_confidence_validates_range_and_type(client):
+    c, state, _ = client
+    assert c.post("/confidence", json={"question_id": "q1", "value": 7}).status_code == 422
+    assert c.post("/confidence", json={"question_id": "q1", "value": "x"}).status_code == 422
+    assert state.confidences == {}
+
+
 def test_publish_calls_post_comment(client):
     c, state, posted = client
     from cognit.engine.models import Results, QuestionResult