diff --git a/src/cognit/engine/models.py b/src/cognit/engine/models.py index c527b33..8d9c047 100644 --- a/src/cognit/engine/models.py +++ b/src/cognit/engine/models.py @@ -108,6 +108,7 @@ class QuestionResult(BaseModel): correct: bool score: int = Field(ge=0, le=100) # 0..100 feedback: str # "" for deterministic questions + confidence: int | None = Field(default=None, ge=1, le=5) # reader's 1–5 self-rating, if given class Results(BaseModel): diff --git a/src/cognit/mcp/assets/quiz_mcp.js b/src/cognit/mcp/assets/quiz_mcp.js index 41e2565..b4efb4d 100644 --- a/src/cognit/mcp/assets/quiz_mcp.js +++ b/src/cognit/mcp/assets/quiz_mcp.js @@ -122,6 +122,16 @@ async function postAnswer(qid, value) { } } +// Persist a confidence rating to the server (fire-and-forget) so the host agent can see +// confidence vs. correctness. Local state is authoritative for the UI; this just mirrors it. +function postConfidence(qid, value) { + fetch("/confidence", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ question_id: qid, value }), + }).catch((e) => console.error("confidence POST failed:", e)); +} + // Re-POST every non-empty local answer (idempotent) so the server has everything // before grading — covers open answers that only POST on blur. async function flushAnswers() { @@ -411,10 +421,59 @@ function isCorrectLocal(q) { return q.type === "tf" ? v === String(q.answer) : v === q.answer; } -// Practice mode: a committed deterministic question reveals immediately and locks. -function shouldReveal(q) { return !examMode && isDeterministic(q) && isAnswered(q); } +// Practice mode, after committing a deterministic question: first rate confidence, +// then reveal. Confidence is local + honor-system (the answer's already in the page). +const confidence = {}; // qid -> 1..5 +const CONFIDENCE_SCALE = [ + [1, "Guessing"], [2, "Unsure"], [3, "Maybe"], [4, "Fairly sure"], [5, "Certain"], +]; +function needsConfidence(q) { + return !examMode && isDeterministic(q) && isAnswered(q) && confidence[q.id] == null; +} +function shouldReveal(q) { + return !examMode && isDeterministic(q) && isAnswered(q) && confidence[q.id] != null; +} + +// Calibration verdict from (confidence, correctness): the teachable case is being +// confident and wrong. Returns null when nothing notable. +function calibration(q) { + const c = confidence[q.id]; + if (c == null) return null; + const correct = isCorrectLocal(q); + if (!correct && c >= 4) return { cls: "over", text: "Confident but wrong — worth a closer look." }; + if (correct && c <= 2) return { cls: "under", text: "Right — but you weren't sure. Lock in why." }; + return null; +} + +function renderConfidencePrompt(q, i) { + const pick = (n) => { confidence[q.id] = n; postConfidence(q.id, n); renderQuestions(); }; + const scale = el("div", { class: "confidence__scale", role: "radiogroup", "aria-label": "Confidence" }, + CONFIDENCE_SCALE.map(([n, label]) => el("button", { + class: "confidence__btn", type: "button", title: label, "aria-label": `${n} — ${label}`, + text: String(n), onclick: () => pick(n), + }))); + return el("article", { class: "file" }, [ + el("div", { class: "file__head" }, [ + el("div", { class: "file__title", text: `Question ${i + 1}` }), + el("div", { class: "file__type", text: TYPE_LABEL[q.type] }), + ]), + el("div", { class: "file__body" }, [ + el("p", { class: "prompt" }, renderPrompt(q.prompt)), + renderAnchor(q), + el("div", { class: "confidence" }, [ + el("div", { class: "confidence__q", text: "How sure are you?" }), + scale, + el("div", { class: "confidence__ends" }, [ + el("span", { text: "1 · guessing" }), + el("span", { text: "5 · certain" }), + ]), + ]), + ]), + ]); +} function renderQuestion(q, i) { + if (needsConfidence(q)) return renderConfidencePrompt(q, i); // rate confidence, then reveal if (shouldReveal(q)) { // Reuse the results card (correct/your-pick rows + explanation), with a locally // computed verdict — no server round-trip, the answer is already in /state. @@ -653,6 +712,14 @@ function renderResultCard(q, r, i) { el("p", { text: q.explanation }), ])); } + if (confidence[q.id] != null) { + const cal = calibration(q); + const label = (CONFIDENCE_SCALE.find(([n]) => n === confidence[q.id]) || [, ""])[1]; + body.push(el("div", { class: "calibration" + (cal ? ` calibration--${cal.cls}` : "") }, [ + el("span", { class: "calibration__rating", text: `Confidence ${confidence[q.id]}/5 · ${label}` }), + cal ? el("span", { class: "calibration__flag", text: cal.text }) : null, + ])); + } return el("article", { class: `file ${cls}` }, [ el("div", { class: "file__head" }, [ el("div", { class: "file__title", text: `Question ${i + 1}` }), diff --git a/src/cognit/mcp/assets/styles.css b/src/cognit/mcp/assets/styles.css index 8d89c93..41ac34d 100644 --- a/src/cognit/mcp/assets/styles.css +++ b/src/cognit/mcp/assets/styles.css @@ -885,3 +885,41 @@ a.codepanel__file:hover { color: var(--blue); text-decoration: underline; } overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .sidelist.coverage li.uncovered .coverage__file { color: var(--fg-mute); } + +/* confidence rating (practice mode, before reveal) ───────────────── */ +.confidence { + border: 1px solid var(--border); + border-radius: 8px; + padding: 16px; + background: var(--bg-subtle); +} +.confidence__q { font-size: 14px; font-weight: 600; color: var(--fg); margin-bottom: 12px; } +.confidence__scale { display: flex; gap: 8px; } +.confidence__btn { + flex: 1; min-width: 0; + padding: 12px 0; + font: 600 15px var(--mono); + color: var(--fg); + background: var(--bg-canvas); + border: 1px solid var(--border); + border-radius: 6px; + cursor: pointer; + transition: border-color 120ms, background 120ms; +} +.confidence__btn:hover { border-color: var(--blue); background: var(--blue-bg); } +.confidence__btn:focus-visible { outline: 2px solid var(--blue); outline-offset: 2px; } +.confidence__ends { display: flex; justify-content: space-between; margin-top: 6px; font-size: 11px; color: var(--fg-faint); } + +/* calibration verdict (shown on the revealed card) */ +.calibration { + display: flex; flex-wrap: wrap; align-items: baseline; gap: 6px 12px; + margin-top: 12px; padding: 8px 12px; + border-radius: 6px; + font-size: 13px; + background: var(--bg-subtle); + color: var(--fg-mute); +} +.calibration__rating { font-family: var(--mono); font-size: 12px; } +.calibration__flag { font-weight: 500; } +.calibration--over { background: var(--red-bg); color: var(--red); } +.calibration--under { background: var(--orange-bg); color: var(--orange); } diff --git a/src/cognit/mcp/grading.py b/src/cognit/mcp/grading.py index 3b3f604..be3adc0 100644 --- a/src/cognit/mcp/grading.py +++ b/src/cognit/mcp/grading.py @@ -20,5 +20,17 @@ def grade_state(state: QuizState, *, llm: LLMClient) -> Results: entries=[AnswerEntry(question_id=qid, value=val) for qid, val in answers_map.items()], ) results = grade(quiz, answers, llm=llm) + # Attach the reader's self-reported confidence (1–5) to each result so the agent can + # see confidence vs. correctness — e.g. to follow up on confident-but-wrong questions. + confidences = dict(state.confidences) + if confidences: + results = results.model_copy( + update={ + "per_question": [ + r.model_copy(update={"confidence": confidences.get(r.question_id)}) + for r in results.per_question + ] + } + ) state.set_results(results) return results diff --git a/src/cognit/mcp/server.py b/src/cognit/mcp/server.py index d165315..1eaddcd 100644 --- a/src/cognit/mcp/server.py +++ b/src/cognit/mcp/server.py @@ -60,7 +60,7 @@ def do_replace_question(state: QuizState, index: int, question: dict[str, Any]) def do_get_answers(state: QuizState) -> dict[str, Any]: snap = state.snapshot() - return {"answers": snap["answers"], "quiz": snap["quiz"]} + return {"answers": snap["answers"], "confidences": snap["confidences"], "quiz": snap["quiz"]} def do_grade(state: QuizState, *, llm: LLMClient) -> dict[str, Any]: @@ -137,7 +137,10 @@ def replace_question(index: int, question: dict[str, Any]) -> dict[str, Any]: @mcp.tool() def get_answers() -> dict[str, Any]: - """Read back the answers the developer selected in the browser + the current quiz.""" + """Read back what the developer did in the browser: their `answers`, their + per-question `confidences` (1–5 self-ratings, where given), and the current quiz. + Confidence vs. correctness is useful for follow-ups — e.g. probe again where they + were confident but wrong, or drill a concept they were unsure about.""" return do_get_answers(state) @mcp.tool() diff --git a/src/cognit/mcp/state.py b/src/cognit/mcp/state.py index 6ba76ad..5763574 100644 --- a/src/cognit/mcp/state.py +++ b/src/cognit/mcp/state.py @@ -21,6 +21,7 @@ def __init__(self, *, pr_number: int, snapshot_path: Path) -> None: self._lock = threading.Lock() self.quiz: Quiz | None = None self.answers: dict[str, str] = {} + self.confidences: dict[str, int] = {} # question_id -> reader's 1–5 self-rating self.results: Results | None = None self._load() @@ -34,6 +35,7 @@ def _load(self) -> None: if data.get("quiz"): self.quiz = Quiz.model_validate(data["quiz"]) self.answers = dict(data.get("answers") or {}) + self.confidences = {k: int(v) for k, v in (data.get("confidences") or {}).items()} if data.get("results"): self.results = Results.model_validate(data["results"]) @@ -41,6 +43,7 @@ def _persist(self) -> None: payload = { "quiz": self.quiz.model_dump(mode="json") if self.quiz else None, "answers": self.answers, + "confidences": self.confidences, "results": self.results.model_dump(mode="json") if self.results else None, } self._snapshot_path.write_text(json.dumps(payload)) @@ -49,6 +52,7 @@ def set_quiz(self, quiz: Quiz) -> None: with self._lock: self.quiz = quiz self.answers = {} + self.confidences = {} self.results = None self._persist() @@ -61,6 +65,7 @@ def replace_question(self, index: int, question: Question) -> None: qs[index] = question self.quiz = self.quiz.model_copy(update={"questions": qs}) self.answers.pop(old_id, None) + self.confidences.pop(old_id, None) self.results = None self._persist() @@ -69,6 +74,11 @@ def record_answer(self, question_id: str, value: str) -> None: self.answers[question_id] = value self._persist() + def record_confidence(self, question_id: str, value: int) -> None: + with self._lock: + self.confidences[question_id] = value + self._persist() + def set_results(self, results: Results) -> None: with self._lock: self.results = results @@ -95,5 +105,6 @@ def snapshot(self) -> dict[str, object]: return { "quiz": self.quiz.model_dump(mode="json") if self.quiz else None, "answers": dict(self.answers), + "confidences": dict(self.confidences), "results": self.results.model_dump(mode="json") if self.results else None, } diff --git a/src/cognit/mcp/web.py b/src/cognit/mcp/web.py index e9b61e8..25ae7d6 100644 --- a/src/cognit/mcp/web.py +++ b/src/cognit/mcp/web.py @@ -1,14 +1,17 @@ """FastAPI app: the browser projection over QuizState. Endpoints: - GET /state — JSON {quiz, answers, results}; the browser polls this - POST /answer — {question_id, value} → record a browser-side answer - POST /grade — human-triggered "Submit quiz": grade now (handler-owned, same path - the agent's `grade` tool uses) and store results. Returns the Results. - POST /publish — human-gated: render + post the results scorecard comment (reuses - ghio.pr.post_comment). The ONLY outward-facing action; never an agent tool. - GET / — the quiz page (polls /state) - GET /static/* — bundled assets + GET /state — JSON {quiz, answers, confidences, results}; the browser polls this + POST /answer — {question_id, value} → record a browser-side answer + POST /confidence — {question_id, value:1-5} → record the reader's confidence rating + GET /diff — ?path= → the unified-diff section for one changed file (inline hunks) + GET /changed-files — JSON {files:[...]} for the diff coverage map + POST /grade — human-triggered "Submit quiz": grade now (handler-owned, same path + the agent's `grade` tool uses) and store results. Returns the Results. + POST /publish — human-gated: render + post the results scorecard comment (reuses + ghio.pr.post_comment). The ONLY outward-facing action; never an agent tool. + GET / — the quiz page (polls /state) + GET /static/* — bundled assets """ from __future__ import annotations @@ -82,6 +85,21 @@ async def post_answer(req: Request) -> JSONResponse: state.record_answer(qid, value) return JSONResponse({"ok": True}) + @app.post("/confidence") + async def post_confidence(req: Request) -> JSONResponse: + body = await req.json() + qid, value = body.get("question_id"), body.get("value") + # bool is an int subclass — reject it explicitly so True/False can't sneak through. + if not isinstance(qid, str) or isinstance(value, bool) or not isinstance(value, int): + return JSONResponse( + {"ok": False, "error": "question_id (string) and value (int) required"}, + status_code=422, + ) + if not (1 <= value <= 5): + return JSONResponse({"ok": False, "error": "value must be 1–5"}, status_code=422) + state.record_confidence(qid, value) + return JSONResponse({"ok": True}) + @app.get("/diff", response_class=PlainTextResponse) def get_diff(path: str = "") -> PlainTextResponse: if diff_section is None: diff --git a/tests/engine/test_models.py b/tests/engine/test_models.py index 67ae4b7..502cdc3 100644 --- a/tests/engine/test_models.py +++ b/tests/engine/test_models.py @@ -111,6 +111,16 @@ def test_objective_questions_carry_optional_explanation(): ) +def test_question_result_confidence_optional(): + r = QuestionResult(question_id="q1", correct=True, score=100, feedback="") + assert r.confidence is None # backward compatible: defaults to None + r2 = QuestionResult(question_id="q1", correct=False, score=0, feedback="", confidence=5) + assert r2.confidence == 5 + for bad in (0, 6): + with pytest.raises(ValidationError): + QuestionResult(question_id="q1", correct=True, score=100, feedback="", confidence=bad) + + def test_anchor_round_trip(): a = Anchor(path="src/cognit/mcp/state.py", start_line=40, end_line=46) assert Anchor.model_validate(a.model_dump()) == a diff --git a/tests/mcp/test_state.py b/tests/mcp/test_state.py index 2c8168b..2640605 100644 --- a/tests/mcp/test_state.py +++ b/tests/mcp/test_state.py @@ -46,6 +46,39 @@ def test_replace_question_drops_old_answer(tmp_path: Path) -> None: assert s.quiz is not None and s.quiz.questions[0].id == "q1b" # new question in place +def test_record_confidence(tmp_path: Path) -> None: + s = QuizState(pr_number=7, snapshot_path=tmp_path / "s.json") + s.set_quiz(_quiz()) + s.record_confidence("q1", 4) + assert s.confidences == {"q1": 4} + + +def test_confidences_persisted_and_reloaded(tmp_path: Path) -> None: + snap = tmp_path / "s.json" + s = QuizState(pr_number=7, snapshot_path=snap) + s.set_quiz(_quiz()) + s.record_confidence("q1", 3) + s2 = QuizState(pr_number=7, snapshot_path=snap) + assert s2.confidences == {"q1": 3} + assert s2.snapshot()["confidences"] == {"q1": 3} + + +def test_set_quiz_resets_confidences(tmp_path: Path) -> None: + s = QuizState(pr_number=7, snapshot_path=tmp_path / "s.json") + s.set_quiz(_quiz()) + s.record_confidence("q1", 5) + s.set_quiz(_quiz()) + assert s.confidences == {} + + +def test_replace_question_drops_confidence(tmp_path: Path) -> None: + s = QuizState(pr_number=7, snapshot_path=tmp_path / "s.json") + s.set_quiz(_quiz()) + s.record_confidence("q1", 5) + s.replace_question(0, MCQQuestion(id="q1b", prompt="p2", options=["X", "Y"], answer="Y")) + assert "q1" not in s.confidences + + def test_corrupt_snapshot_loads_as_empty(tmp_path: Path) -> None: snap = tmp_path / "s.json" snap.write_text("{not valid json") diff --git a/tests/mcp/test_tools.py b/tests/mcp/test_tools.py index e180562..d0bc04a 100644 --- a/tests/mcp/test_tools.py +++ b/tests/mcp/test_tools.py @@ -88,6 +88,25 @@ def test_get_answers(tmp_path: Path): assert out["quiz"]["questions"][0]["id"] == "q1" +def test_get_answers_includes_confidences(tmp_path: Path): + state = _state(tmp_path) + srv.do_set_quiz(state, _draft()) + state.record_answer("q1", "A") + state.record_confidence("q1", 2) + out = srv.do_get_answers(state) + assert out["confidences"] == {"q1": 2} + + +def test_grade_attaches_confidence(tmp_path: Path): + state = _state(tmp_path) + srv.do_set_quiz(state, _draft()) + state.record_answer("q1", "A") + state.record_confidence("q1", 4) + out = srv.do_grade(state, llm=FakeLLM()) + assert out["ok"] is True + assert out["per_question"][0]["confidence"] == 4 + + def test_grade_without_quiz_returns_structured_failure(tmp_path: Path): out = srv.do_grade(_state(tmp_path), llm=FakeLLM()) assert out["ok"] is False diff --git a/tests/mcp/test_web.py b/tests/mcp/test_web.py index cd97dc2..0eeee8c 100644 --- a/tests/mcp/test_web.py +++ b/tests/mcp/test_web.py @@ -72,6 +72,19 @@ def test_post_answer_records(client): assert state.answers == {"q1": "A"} +def test_post_confidence_records(client): + c, state, _ = client + assert c.post("/confidence", json={"question_id": "q1", "value": 3}).status_code == 200 + assert state.confidences == {"q1": 3} + + +def test_post_confidence_validates_range_and_type(client): + c, state, _ = client + assert c.post("/confidence", json={"question_id": "q1", "value": 7}).status_code == 422 + assert c.post("/confidence", json={"question_id": "q1", "value": "x"}).status_code == 422 + assert state.confidences == {} + + def test_publish_calls_post_comment(client): c, state, posted = client from cognit.engine.models import Results, QuestionResult