From e2208827b660faa7d3fc1eab2271a8eae18526ee Mon Sep 17 00:00:00 2001 From: Scott Converse Date: Sat, 20 Jun 2026 09:26:30 -0600 Subject: [PATCH] Fix records response local AI generation --- backend/app/requests/router.py | 94 ++++++++++++++++++++--- backend/tests/test_response_letter.py | 106 ++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 12 deletions(-) diff --git a/backend/app/requests/router.py b/backend/app/requests/router.py index 2f745f1..3ba6dda 100644 --- a/backend/app/requests/router.py +++ b/backend/app/requests/router.py @@ -1,8 +1,11 @@ +import json +import os import uuid from datetime import datetime, timezone, timedelta import logging +import httpx from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy import select, func @@ -914,6 +917,10 @@ async def review_fee_waiver( "This letter was generated by CivicRecords AI and must be reviewed " "and approved by authorized staff before being sent to the requester." ) +_LOCAL_GENERATION_NUM_PREDICT = 220 +_LOCAL_GENERATION_NUM_CTX = 3072 +_LOCAL_GENERATION_TIMEOUT_SECONDS = 120.0 +_GEMMA_STOP_TOKENS = ["", ""] def _generate_template_letter( @@ -989,13 +996,81 @@ def _generate_template_letter( return "\n".join(lines) + _LETTER_DISCLAIMER +def _records_runtime_model() -> str: + return os.environ.get("LLM_MODEL") or settings.chat_model + + +def _records_gemma_prompt(prompt: str) -> str: + bounded_prompt = ( + f"{prompt}\n\n" + "Return a concise staff-review records response draft in under 220 words. " + "Use plain text only. Do not include hidden reasoning or analysis." + ) + return f"user\n{bounded_prompt}\nmodel\n" + + +def _records_generation_payload(prompt: str) -> dict: + return { + "model": _records_runtime_model(), + "prompt": _records_gemma_prompt(prompt), + "raw": True, + "stream": False, + "options": { + "temperature": 0.2, + "num_predict": _LOCAL_GENERATION_NUM_PREDICT, + "num_ctx": _LOCAL_GENERATION_NUM_CTX, + "stop": _GEMMA_STOP_TOKENS, + }, + } + + +def _parse_ollama_generate_text(body: str) -> str: + """Parse Ollama generate variants seen from product-managed local runtimes.""" + fragments: list[str] = [] + for line in body.splitlines(): + candidate = line.strip() + if not candidate: + continue + # Defensive tolerance for callers that surface chunk-size lines before + # JSON decoding; httpx normally removes these before we parse. + try: + int(candidate, 16) + continue + except ValueError: + pass + try: + data = json.loads(candidate) + except json.JSONDecodeError: + continue + response = data.get("response") + if isinstance(response, str): + fragments.append(response) + message = data.get("message") + if isinstance(message, dict) and isinstance(message.get("content"), str): + fragments.append(message["content"]) + + if fragments: + return "".join(fragments).strip() + + try: + data = json.loads(body) + except json.JSONDecodeError: + return "" + response = data.get("response") + if isinstance(response, str): + return response.strip() + message = data.get("message") + if isinstance(message, dict) and isinstance(message.get("content"), str): + return message["content"].strip() + return "" + + async def _try_llm_generation( req: "RecordsRequest", docs: list[tuple["RequestDocument", "Document"]], ) -> str | None: """Attempt to generate a letter via Ollama. Returns None on failure.""" try: - import httpx from app.llm.context_manager import ( assemble_context, blocks_to_prompt, @@ -1048,20 +1123,15 @@ async def _try_llm_generation( ) prompt = blocks_to_prompt(blocks) - async with httpx.AsyncClient(timeout=60.0) as client: + async with httpx.AsyncClient(timeout=_LOCAL_GENERATION_TIMEOUT_SECONDS) as client: resp = await client.post( f"{settings.ollama_base_url}/api/generate", - json={ - "model": settings.chat_model, - "prompt": prompt, - "stream": False, - }, + json=_records_generation_payload(prompt), ) - if resp.status_code == 200: - data = resp.json() - generated = data.get("response", "") - if generated.strip(): - return generated.strip() + _LETTER_DISCLAIMER + resp.raise_for_status() + generated = _parse_ollama_generate_text(resp.text) + if generated: + return generated + _LETTER_DISCLAIMER except Exception as exc: logger.warning("LLM generation failed, falling back to template: %s", exc) diff --git a/backend/tests/test_response_letter.py b/backend/tests/test_response_letter.py index f983cee..f6c7018 100644 --- a/backend/tests/test_response_letter.py +++ b/backend/tests/test_response_letter.py @@ -1,6 +1,112 @@ +import uuid +from datetime import datetime, timezone +from types import SimpleNamespace + import pytest from httpx import AsyncClient +from app.requests.router import ( + _LETTER_DISCLAIMER, + _parse_ollama_generate_text, + _try_llm_generation, + _records_generation_payload, +) + + +def test_records_generation_payload_uses_pinned_runtime_model(monkeypatch): + monkeypatch.setenv("LLM_MODEL", "civicsuite-gemma4-12b-qat:q4_0") + + payload = _records_generation_payload("Draft a response letter.") + + assert payload["model"] == "civicsuite-gemma4-12b-qat:q4_0" + assert payload["raw"] is True + assert payload["stream"] is False + assert "user" in payload["prompt"] + assert "model" in payload["prompt"] + assert payload["options"]["num_predict"] <= 220 + assert payload["options"]["num_ctx"] <= 3072 + assert "" in payload["options"]["stop"] + + +def test_parse_ollama_generate_text_accepts_response_field(): + body = '{"response":"D105-AI-MODEL-MARKER-20260620 records draft."}' + + assert ( + _parse_ollama_generate_text(body) + == "D105-AI-MODEL-MARKER-20260620 records draft." + ) + + +def test_parse_ollama_generate_text_accepts_chat_message_shape(): + body = '{"message":{"content":"D105-AI-MODEL-MARKER-20260620 chat draft."}}' + + assert ( + _parse_ollama_generate_text(body) + == "D105-AI-MODEL-MARKER-20260620 chat draft." + ) + + +def test_parse_ollama_generate_text_accepts_chunked_json_lines(): + body = ( + "94\r\n" + '{"response":"D105-AI-MODEL-"}\n' + "11\r\n" + '{"response":"MARKER-20260620 records draft."}\n' + "0\r\n" + ) + + assert ( + _parse_ollama_generate_text(body) + == "D105-AI-MODEL-MARKER-20260620 records draft." + ) + + +@pytest.mark.asyncio +async def test_try_llm_generation_returns_local_ai_letter(monkeypatch): + captured: dict[str, object] = {} + + class FakeResponse: + text = '{"response":"D105-AI-MODEL-MARKER-20260620 local records letter."}' + + def raise_for_status(self): + return None + + class FakeAsyncClient: + def __init__(self, *, timeout): + captured["timeout"] = timeout + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def post(self, url, *, json): + captured["url"] = url + captured["json"] = json + return FakeResponse() + + monkeypatch.setenv("LLM_MODEL", "civicsuite-gemma4-12b-qat:q4_0") + monkeypatch.setattr("app.requests.router.httpx.AsyncClient", FakeAsyncClient) + + req = SimpleNamespace( + requester_name="D105 Records Requester", + id=uuid.uuid4(), + date_received=datetime.now(timezone.utc), + description="Please provide D105-AI-MODEL-MARKER-20260620 records.", + ) + + generated = await _try_llm_generation(req, []) + + assert generated is not None + assert "D105-AI-MODEL-MARKER-20260620 local records letter." in generated + assert _LETTER_DISCLAIMER in generated + payload = captured["json"] + assert payload["model"] == "civicsuite-gemma4-12b-qat:q4_0" + assert payload["raw"] is True + assert payload["stream"] is False + assert "D105-AI-MODEL-MARKER-20260620" in payload["prompt"] + @pytest.mark.asyncio async def test_generate_response_letter(client: AsyncClient, admin_token: str):