Release the Windows-Local AI records-response fix bundled in CivicSuite v1.0.2

backend/app/requests/router.py

@@ -1,8 +1,11 @@
+import json
+import os
 import uuid
 from datetime import datetime, timezone, timedelta
 
 import logging
 
+import httpx
 from fastapi import APIRouter, Depends, HTTPException
 from pydantic import BaseModel
 from sqlalchemy import select, func
@@ -914,6 +917,10 @@ async def review_fee_waiver(
     "This letter was generated by CivicRecords AI and must be reviewed "
     "and approved by authorized staff before being sent to the requester."
 )
+_LOCAL_GENERATION_NUM_PREDICT = 220
+_LOCAL_GENERATION_NUM_CTX = 3072
+_LOCAL_GENERATION_TIMEOUT_SECONDS = 120.0
+_GEMMA_STOP_TOKENS = ["<end_of_turn>", "<start_of_turn>"]
 
 
 def _generate_template_letter(
@@ -989,13 +996,81 @@ def _generate_template_letter(
     return "\n".join(lines) + _LETTER_DISCLAIMER
 
 
+def _records_runtime_model() -> str:
+    return os.environ.get("LLM_MODEL") or settings.chat_model
+
+
+def _records_gemma_prompt(prompt: str) -> str:
+    bounded_prompt = (
+        f"{prompt}\n\n"
+        "Return a concise staff-review records response draft in under 220 words. "
+        "Use plain text only. Do not include hidden reasoning or analysis."
+    )
+    return f"<start_of_turn>user\n{bounded_prompt}<end_of_turn>\n<start_of_turn>model\n"
+
+
+def _records_generation_payload(prompt: str) -> dict:
+    return {
+        "model": _records_runtime_model(),
+        "prompt": _records_gemma_prompt(prompt),
+        "raw": True,
+        "stream": False,
+        "options": {
+            "temperature": 0.2,
+            "num_predict": _LOCAL_GENERATION_NUM_PREDICT,
+            "num_ctx": _LOCAL_GENERATION_NUM_CTX,
+            "stop": _GEMMA_STOP_TOKENS,
+        },
+    }
+
+
+def _parse_ollama_generate_text(body: str) -> str:
+    """Parse Ollama generate variants seen from product-managed local runtimes."""
+    fragments: list[str] = []
+    for line in body.splitlines():
+        candidate = line.strip()
+        if not candidate:
+            continue
+        # Defensive tolerance for callers that surface chunk-size lines before
+        # JSON decoding; httpx normally removes these before we parse.
+        try:
+            int(candidate, 16)
+            continue
+        except ValueError:
+            pass
+        try:
+            data = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        response = data.get("response")
+        if isinstance(response, str):
+            fragments.append(response)
+        message = data.get("message")
+        if isinstance(message, dict) and isinstance(message.get("content"), str):
+            fragments.append(message["content"])
+
+    if fragments:
+        return "".join(fragments).strip()
+
+    try:
+        data = json.loads(body)
+    except json.JSONDecodeError:
+        return ""
+    response = data.get("response")
+    if isinstance(response, str):
+        return response.strip()
+    message = data.get("message")
+    if isinstance(message, dict) and isinstance(message.get("content"), str):
+        return message["content"].strip()
+    return ""
+
+
 async def _try_llm_generation(
     req: "RecordsRequest",
     docs: list[tuple["RequestDocument", "Document"]],
 ) -> str | None:
     """Attempt to generate a letter via Ollama. Returns None on failure."""
     try:
-        import httpx
         from app.llm.context_manager import (
             assemble_context,
             blocks_to_prompt,
@@ -1048,20 +1123,15 @@ async def _try_llm_generation(
         )
         prompt = blocks_to_prompt(blocks)
 
-        async with httpx.AsyncClient(timeout=60.0) as client:
+        async with httpx.AsyncClient(timeout=_LOCAL_GENERATION_TIMEOUT_SECONDS) as client:
             resp = await client.post(
                 f"{settings.ollama_base_url}/api/generate",
-                json={
-                    "model": settings.chat_model,
-                    "prompt": prompt,
-                    "stream": False,
-                },
+                json=_records_generation_payload(prompt),
             )
-            if resp.status_code == 200:
-                data = resp.json()
-                generated = data.get("response", "")
-                if generated.strip():
-                    return generated.strip() + _LETTER_DISCLAIMER
+            resp.raise_for_status()
+            generated = _parse_ollama_generate_text(resp.text)
+            if generated:
+                return generated + _LETTER_DISCLAIMER
     except Exception as exc:
         logger.warning("LLM generation failed, falling back to template: %s", exc)
 

backend/tests/test_response_letter.py

@@ -1,6 +1,112 @@
+import uuid
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
 import pytest
 from httpx import AsyncClient
 
+from app.requests.router import (
+    _LETTER_DISCLAIMER,
+    _parse_ollama_generate_text,
+    _try_llm_generation,
+    _records_generation_payload,
+)
+
+
+def test_records_generation_payload_uses_pinned_runtime_model(monkeypatch):
+    monkeypatch.setenv("LLM_MODEL", "civicsuite-gemma4-12b-qat:q4_0")
+
+    payload = _records_generation_payload("Draft a response letter.")
+
+    assert payload["model"] == "civicsuite-gemma4-12b-qat:q4_0"
+    assert payload["raw"] is True
+    assert payload["stream"] is False
+    assert "<start_of_turn>user" in payload["prompt"]
+    assert "<start_of_turn>model" in payload["prompt"]
+    assert payload["options"]["num_predict"] <= 220
+    assert payload["options"]["num_ctx"] <= 3072
+    assert "<end_of_turn>" in payload["options"]["stop"]
+
+
+def test_parse_ollama_generate_text_accepts_response_field():
+    body = '{"response":"D105-AI-MODEL-MARKER-20260620 records draft."}'
+
+    assert (
+        _parse_ollama_generate_text(body)
+        == "D105-AI-MODEL-MARKER-20260620 records draft."
+    )
+
+
+def test_parse_ollama_generate_text_accepts_chat_message_shape():
+    body = '{"message":{"content":"D105-AI-MODEL-MARKER-20260620 chat draft."}}'
+
+    assert (
+        _parse_ollama_generate_text(body)
+        == "D105-AI-MODEL-MARKER-20260620 chat draft."
+    )
+
+
+def test_parse_ollama_generate_text_accepts_chunked_json_lines():
+    body = (
+        "94\r\n"
+        '{"response":"D105-AI-MODEL-"}\n'
+        "11\r\n"
+        '{"response":"MARKER-20260620 records draft."}\n'
+        "0\r\n"
+    )
+
+    assert (
+        _parse_ollama_generate_text(body)
+        == "D105-AI-MODEL-MARKER-20260620 records draft."
+    )
+
+
+@pytest.mark.asyncio
+async def test_try_llm_generation_returns_local_ai_letter(monkeypatch):
+    captured: dict[str, object] = {}
+
+    class FakeResponse:
+        text = '{"response":"D105-AI-MODEL-MARKER-20260620 local records letter."}'
+
+        def raise_for_status(self):
+            return None
+
+    class FakeAsyncClient:
+        def __init__(self, *, timeout):
+            captured["timeout"] = timeout
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, exc_type, exc, tb):
+            return None
+
+        async def post(self, url, *, json):
+            captured["url"] = url
+            captured["json"] = json
+            return FakeResponse()
+
+    monkeypatch.setenv("LLM_MODEL", "civicsuite-gemma4-12b-qat:q4_0")
+    monkeypatch.setattr("app.requests.router.httpx.AsyncClient", FakeAsyncClient)
+
+    req = SimpleNamespace(
+        requester_name="D105 Records Requester",
+        id=uuid.uuid4(),
+        date_received=datetime.now(timezone.utc),
+        description="Please provide D105-AI-MODEL-MARKER-20260620 records.",
+    )
+
+    generated = await _try_llm_generation(req, [])
+
+    assert generated is not None
+    assert "D105-AI-MODEL-MARKER-20260620 local records letter." in generated
+    assert _LETTER_DISCLAIMER in generated
+    payload = captured["json"]
+    assert payload["model"] == "civicsuite-gemma4-12b-qat:q4_0"
+    assert payload["raw"] is True
+    assert payload["stream"] is False
+    assert "D105-AI-MODEL-MARKER-20260620" in payload["prompt"]
+
 
 @pytest.mark.asyncio
 async def test_generate_response_letter(client: AsyncClient, admin_token: str):