feat: implement Master Incident Data Lake - Supports File Once, Report Everywhere (along with Report Once , File Everywhere)

.gitignore

@@ -2,4 +2,12 @@
 .idea
 venv
 .venv
-*.db
+*.dbsrc/inputs/*.pdf
+src/outputs/*.pdf
+src/inputs/*.pdf
+src/outputs/*.pdf
+fireform.db 
+*.bak 
+ngrok.exe 
+out.txt 
+benchmark_proof.py 

api/db/models.py

@@ -15,4 +15,28 @@ class FormSubmission(SQLModel, table=True):
     template_id: int
     input_text: str
     output_pdf_path: str
-    created_at: datetime = Field(default_factory=datetime.utcnow)
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+# ADD THIS TO api/db/models.py
+# (append to existing file — don't replace)
+
+from sqlmodel import SQLModel, Field
+from typing import Optional
+from datetime import datetime
+
+
+class IncidentMasterData(SQLModel, table=True):
+    """
+    The Incident Data Lake.
+    Stores all extracted data from one incident as a master JSON blob.
+    Any agency can generate their PDF from this single record — zero new LLM calls.
+    """
+    id: Optional[int] = Field(default=None, primary_key=True)
+    incident_id: str = Field(index=True)        # INC-2026-0321-4821
+    master_json: str                             # JSON string — all extracted fields
+    transcript_text: str                         # original transcript
+    location_lat: Optional[float] = None        # from PWA GPS
+    location_lng: Optional[float] = None        # from PWA GPS
+    officer_notes: Optional[str] = None         # additional context
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: datetime = Field(default_factory=datetime.utcnow)

api/db/repositories.py

@@ -1,19 +1,117 @@
 from sqlmodel import Session, select
 from api.db.models import Template, FormSubmission
 
-# Templates
+
+# ── Templates ─────────────────────────────────────────────────
+
 def create_template(session: Session, template: Template) -> Template:
     session.add(template)
     session.commit()
     session.refresh(template)
     return template
 
+
 def get_template(session: Session, template_id: int) -> Template | None:
     return session.get(Template, template_id)
 
-# Forms
+
+def get_all_templates(session: Session, limit: int = 100, offset: int = 0) -> list[Template]:
+    statement = select(Template).offset(offset).limit(limit)
+    return session.exec(statement).all()
+
+
+# ── Forms ─────────────────────────────────────────────────────
+
 def create_form(session: Session, form: FormSubmission) -> FormSubmission:
     session.add(form)
     session.commit()
     session.refresh(form)
-    return form
+    return form
+
+
+def get_form(session: Session, submission_id: int) -> FormSubmission | None:
+    return session.get(FormSubmission, submission_id)
+
+
+# ADD THESE FUNCTIONS TO api/db/repositories.py
+# (append to existing file — don't replace)
+
+import json
+from api.db.models import IncidentMasterData
+from datetime import datetime
+
+
+def create_incident(db, incident: IncidentMasterData) -> IncidentMasterData:
+    db.add(incident)
+    db.commit()
+    db.refresh(incident)
+    return incident
+
+
+def get_incident(db, incident_id: str) -> IncidentMasterData:
+    from sqlmodel import select
+    return db.exec(
+        select(IncidentMasterData).where(
+            IncidentMasterData.incident_id == incident_id
+        )
+    ).first()
+
+
+def get_all_incidents(db) -> list:
+    from sqlmodel import select
+    return db.exec(select(IncidentMasterData)).all()
+
+
+def update_incident_json(db, incident_id: str, new_data: dict, new_transcript: str = None) -> IncidentMasterData:
+    """
+    Smart Merge new extracted data into existing master JSON to enable 
+    Collaborative Incident Consensus. Protects existing data from being 
+    wiped by LLM `null` hallucinations, and appends long-form text.
+    """
+    incident = get_incident(db, incident_id)
+    if not incident:
+        return None
+
+    existing = json.loads(incident.master_json)
+
+    for key, value in new_data.items():
+        # 1. Ignore empty/null values to protect existing data
+        if value is None or str(value).strip().lower() in ("null", "none", "", "n/a"):
+            continue
+
+        # 2. If the field exists, handle smart merging vs overwriting
+        if key in existing and existing[key]:
+            old_value = existing[key]
+
+            # Use string representation for safe comparison
+            old_str = str(old_value).strip() if not isinstance(old_value, list) else "\n".join(str(i) for i in old_value)
+            new_str = str(value).strip() if not isinstance(value, list) else "\n".join(str(i) for i in value)
+
+            # If the value is identical, do nothing
+            if old_str.lower() == new_str.lower():
+                continue
+
+            # If it's a long-form text field (Notes, Description, Narrative, Summary, etc)
+            long_fields = ("note", "desc", "narrative", "summary", "remark", "detail", "comment")
+            if any(lf in key.lower() for lf in long_fields):
+                # Prevent recursive appending
+                if new_str not in old_str:
+                    existing[key] = f"{old_str}\n\n[UPDATE]: {new_str}"
+            else:
+                # Standard Field Correction (e.g. ID, City) - overwrite the old value
+                existing[key] = value
+        else:
+            # 3. Brand new field
+            existing[key] = value
+
+    incident.master_json = json.dumps(existing)
+
+    # Safely append the new transcript segment for true consensus history
+    if new_transcript and new_transcript.strip() not in incident.transcript_text:
+        incident.transcript_text = f"{incident.transcript_text}\n\n---\n[UPDATE]: {new_transcript.strip()}"
+
+    incident.updated_at = datetime.utcnow()
+    db.add(incident)
+    db.commit()
+    db.refresh(incident)
+    return incident

api/main.py

@@ -1,7 +1,32 @@
-from fastapi import FastAPI
-from api.routes import templates, forms
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from api.routes import templates, forms, transcribe, incidents
+from api.errors.base import AppError
+from typing import Union
+import os
 
 app = FastAPI()
 
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.exception_handler(AppError)
+def app_error_handler(request: Request, exc: AppError):
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"detail": exc.message}
+    )
+
 app.include_router(templates.router)
-app.include_router(forms.router)
+app.include_router(forms.router)
+app.include_router(transcribe.router)
+app.include_router(incidents.router)
+
+if os.path.exists("mobile"):
+    app.mount("/mobile", StaticFiles(directory="mobile", html=True), name="mobile")

api/routes/forms.py

@@ -1,25 +1,140 @@
+import os
 from fastapi import APIRouter, Depends
+from fastapi.responses import FileResponse
 from sqlmodel import Session
 from api.deps import get_db
-from api.schemas.forms import FormFill, FormFillResponse
-from api.db.repositories import create_form, get_template
+from api.schemas.forms import FormFill, FormFillResponse, BatchFormFill, BatchFormFillResponse, BatchResultItem
+from api.db.repositories import create_form, get_template, get_form
 from api.db.models import FormSubmission
 from api.errors.base import AppError
 from src.controller import Controller
+from src.llm import LLM
+from src.filler import Filler
 
 router = APIRouter(prefix="/forms", tags=["forms"])
 
+
 @router.post("/fill", response_model=FormFillResponse)
-def fill_form(form: FormFill, db: Session = Depends(get_db)):
-    if not get_template(db, form.template_id):
+async def fill_form(form: FormFill, db: Session = Depends(get_db)):
+    template = get_template(db, form.template_id)
+    if not template:
         raise AppError("Template not found", status_code=404)
 
-    fetched_template = get_template(db, form.template_id)
+    if not os.path.exists(template.pdf_path):
+        raise AppError(f"Template PDF not found: {template.pdf_path}", status_code=404)
+
+    try:
+        # Step 1: LLM Extraction (Async)
+        llm = LLM(transcript_text=form.input_text, target_fields=template.fields)
+        await llm.async_main_loop()
+        extracted_data = llm.get_data()
 
-    controller = Controller()
-    path = controller.fill_form(user_input=form.input_text, fields=fetched_template.fields, pdf_form_path=fetched_template.pdf_path)
+        # Step 2: PDF Filling (Sync)
+        # Using filler directly to avoid redundant extraction in controller
+        filler = Filler()
+        path = filler.fill_form_with_data(
+            pdf_form=template.pdf_path,
+            data=extracted_data
+        )
+    except Exception as e:
+        raise AppError(f"Processing failed: {str(e)}", status_code=500)
+
+    if not path or not os.path.exists(path):
+        raise AppError("PDF generation failed.", status_code=500)
 
     submission = FormSubmission(**form.model_dump(), output_pdf_path=path)
     return create_form(db, submission)
 
 
+@router.post("/fill/batch", response_model=BatchFormFillResponse)
+async def fill_batch(batch: BatchFormFill, db: Session = Depends(get_db)):
+    if not batch.template_ids:
+        raise AppError("template_ids must not be empty", status_code=400)
+
+    templates = []
+    for tid in batch.template_ids:
+        tpl = get_template(db, tid)
+        if not tpl or not os.path.exists(tpl.pdf_path):
+            raise AppError(f"Template {tid} invalid or PDF missing", status_code=404)
+        templates.append(tpl)
+
+    # Step 1: LLM Extraction (Async - ONE call for all templates)
+    merged_fields = {}
+    for tpl in templates:
+        if isinstance(tpl.fields, dict): merged_fields.update(tpl.fields)
+        else:
+            for f in tpl.fields: merged_fields[f] = f
+
+    try:
+        llm = LLM(transcript_text=batch.input_text, target_fields=merged_fields)
+        await llm.async_main_loop()
+        extracted_json = llm.get_data()
+    except Exception as e:
+        raise AppError(f"Extraction failed: {str(e)}", status_code=500)
+
+    # Step 2: PDF Filling (Sync - per template)
+    results = []
+    success_count = 0
+    filler = Filler()
+
+    for tpl in templates:
+        try:
+            tpl_field_keys = list(tpl.fields.keys()) if isinstance(tpl.fields, dict) else tpl.fields
+            tpl_data = {k: extracted_json.get(k) for k in tpl_field_keys}
+
+            output_path = filler.fill_form_with_data(pdf_form=tpl.pdf_path, data=tpl_data)
+
+            submission = FormSubmission(
+                template_id=tpl.id, 
+                input_text=batch.input_text, 
+                output_pdf_path=output_path
+            )
+            saved = create_form(db, submission)
+
+            results.append(BatchResultItem(
+                template_id=tpl.id, 
+                template_name=tpl.name, 
+                success=True,
+                submission_id=saved.id, 
+                download_url=f"/forms/download/{saved.id}"
+            ))
+            success_count += 1
+        except Exception as e:
+            results.append(BatchResultItem(
+                template_id=tpl.id, 
+                template_name=tpl.name, 
+                success=False, 
+                error=str(e)
+            ))
+
+    return BatchFormFillResponse(
+        total=len(templates), 
+        succeeded=success_count, 
+        failed=len(templates)-success_count, 
+        results=results
+    )
+
+
+@router.get("/{submission_id}", response_model=FormFillResponse)
+def get_submission(submission_id: int, db: Session = Depends(get_db)):
+    submission = get_form(db, submission_id)
+    if not submission:
+        raise AppError("Submission not found", status_code=404)
+    return submission
+
+
+@router.get("/download/{submission_id}")
+def download_filled_pdf(submission_id: int, db: Session = Depends(get_db)):
+    submission = get_form(db, submission_id)
+    if not submission:
+        raise AppError("Submission not found", status_code=404)
+
+    file_path = submission.output_pdf_path
+    if not os.path.exists(file_path):
+        raise AppError("PDF file not found on server", status_code=404)
+
+    return FileResponse(
+        path=file_path,
+        media_type="application/pdf",
+        filename=os.path.basename(file_path)
+    )