feat: add API key auth, health checks, WebSocket, Zustand stores, integration tests, fix notification enum

- Add API key authentication middleware with dev-mode bypass
- Implement real readiness checks (DB + Redis connectivity)
- Add WebSocket real-time event broadcasting via /ws endpoint
- Create Zustand stores for alerts, incidents, stats, silences, notifications, and WS
- Refactor frontend hooks to thin wrappers around Zustand stores
- Rewrite App.tsx to use store-driven state management
- Add 34 integration tests covering all API endpoints
- Fix notification channel enum bug (PostgreSQL expected lowercase values)
- Add WebSocket event emission to alert/incident service operations

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: captivit

.env.example

@@ -17,6 +17,7 @@ REDIS_URL=redis://localhost:6379/0
 # Application
 # ──────────────────────────────────────
 SECRET_KEY=change-me-to-a-random-secret-key
+API_KEY=
 APP_ENV=development
 LOG_LEVEL=INFO
 API_PREFIX=/api/v1

backend/api/deps.py

@@ -0,0 +1,33 @@
+"""Shared API dependencies (auth, etc.)."""
+
+import secrets
+
+from fastapi import HTTPException, Security
+from fastapi.security import APIKeyHeader
+
+from backend.config import get_settings
+
+settings = get_settings()
+
+_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+
+async def require_api_key(
+    api_key: str | None = Security(_api_key_header),
+) -> str:
+    """Validate the API key from the X-API-Key header.
+
+    In development mode with the default secret key, auth is skipped
+    to avoid friction during local development.
+    """
+    # Skip auth in dev mode when using the default placeholder key
+    if settings.is_dev and settings.api_key == "":
+        return "dev"
+
+    if not api_key:
+        raise HTTPException(status_code=401, detail="Missing API key")
+
+    if not secrets.compare_digest(api_key, settings.api_key):
+        raise HTTPException(status_code=403, detail="Invalid API key")
+
+    return api_key

backend/api/routes/health.py

@@ -1,7 +1,17 @@
+import logging
+
 from fastapi import APIRouter
+from sqlalchemy import text
+
+from backend.config import get_settings
+from backend.database import async_session
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(tags=["health"])
 
+settings = get_settings()
+
 
 @router.get("/health")
 async def health_check() -> dict:
@@ -12,5 +22,37 @@ async def health_check() -> dict:
 @router.get("/health/ready")
 async def readiness_check() -> dict:
     """Readiness check — verifies dependencies are available."""
-    # TODO: Check DB and Redis connectivity
-    return {"status": "ready"}
+    checks: dict[str, str] = {}
+
+    # Check database
+    try:
+        async with async_session() as session:
+            await session.execute(text("SELECT 1"))
+        checks["database"] = "ok"
+    except Exception as e:
+        logger.warning(f"Database readiness check failed: {e}")
+        checks["database"] = "unavailable"
+
+    # Check Redis
+    try:
+        import redis.asyncio as aioredis
+
+        r = aioredis.from_url(settings.redis_url, socket_connect_timeout=2)
+        await r.ping()
+        await r.aclose()
+        checks["redis"] = "ok"
+    except Exception as e:
+        logger.warning(f"Redis readiness check failed: {e}")
+        checks["redis"] = "unavailable"
+
+    all_ok = all(v == "ok" for v in checks.values())
+
+    if not all_ok:
+        from fastapi.responses import JSONResponse
+
+        return JSONResponse(
+            status_code=503,
+            content={"status": "degraded", "checks": checks},
+        )
+
+    return {"status": "ready", "checks": checks}

backend/api/routes/ws.py

@@ -0,0 +1,96 @@
+"""WebSocket endpoint for real-time alert/incident updates."""
+
+import json
+import logging
+import secrets
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from backend.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["websocket"])
+
+settings = get_settings()
+
+
+class ConnectionManager:
+    """Manages active WebSocket connections and broadcasts events."""
+
+    def __init__(self) -> None:
+        self._connections: set[WebSocket] = set()
+
+    async def connect(self, ws: WebSocket) -> None:
+        await ws.accept()
+        self._connections.add(ws)
+        logger.info(f"WebSocket connected ({len(self._connections)} active)")
+
+    def disconnect(self, ws: WebSocket) -> None:
+        self._connections.discard(ws)
+        logger.info(f"WebSocket disconnected ({len(self._connections)} active)")
+
+    async def broadcast(self, event: dict) -> None:
+        """Send an event to all connected clients."""
+        if not self._connections:
+            return
+        payload = json.dumps(event)
+        dead: list[WebSocket] = []
+        for ws in self._connections:
+            try:
+                await ws.send_text(payload)
+            except Exception:
+                dead.append(ws)
+        for ws in dead:
+            self._connections.discard(ws)
+
+
+manager = ConnectionManager()
+
+
+def _check_ws_auth(api_key: str | None) -> bool:
+    """Validate API key for WebSocket connections."""
+    if settings.is_dev and settings.api_key == "":
+        return True
+    if not api_key:
+        return False
+    return secrets.compare_digest(api_key, settings.api_key)
+
+
+@router.websocket("/ws")
+async def websocket_endpoint(ws: WebSocket) -> None:
+    """Real-time event stream.
+
+    Clients connect and receive JSON events when alerts or incidents
+    are created, updated, or resolved.
+
+    Auth: pass API key as ?token= query param.
+
+    Event format:
+        {"type": "alert.created", "data": {...}}
+        {"type": "incident.updated", "data": {...}}
+    """
+    token = ws.query_params.get("token")
+    if not _check_ws_auth(token):
+        await ws.close(code=4003, reason="Invalid or missing API key")
+        return
+
+    await manager.connect(ws)
+    try:
+        while True:
+            # Keep connection alive; clients can send pings
+            data = await ws.receive_text()
+            if data == "ping":
+                await ws.send_text(json.dumps({"type": "pong"}))
+    except WebSocketDisconnect:
+        pass
+    finally:
+        manager.disconnect(ws)
+
+
+async def emit_event(event_type: str, data: dict) -> None:
+    """Broadcast an event to all connected WebSocket clients.
+
+    Call this from services when state changes occur.
+    """
+    await manager.broadcast({"type": event_type, "data": data})

backend/config.py

@@ -18,6 +18,7 @@ class Settings(BaseSettings):
     log_level: str = "INFO"
     api_prefix: str = "/api/v1"
     secret_key: str = "change-me-to-a-random-secret-key"
+    api_key: str = ""
 
     # ── Database ─────────────────────────────────────────
     database_url: str = "postgresql+asyncpg://solace:solace@localhost:5432/solace"

backend/main.py

@@ -1,9 +1,11 @@
 import logging
 
-from fastapi import FastAPI
+from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from backend.api.deps import require_api_key
 from backend.api.routes import alerts, health, incidents, notifications, silences, stats, webhooks
+from backend.api.routes.ws import router as ws_router
 from backend.config import get_settings
 
 settings = get_settings()
@@ -40,16 +42,20 @@
 
 # ─── Routes ──────────────────────────────────────────────
 
-# Health checks at root level (no prefix)
+# Health checks at root level (no prefix, no auth — used by k8s probes)
 app.include_router(health.router)
 
-# API routes under /api/v1
-app.include_router(webhooks.router, prefix=settings.api_prefix)
-app.include_router(alerts.router, prefix=settings.api_prefix)
-app.include_router(incidents.router, prefix=settings.api_prefix)
-app.include_router(stats.router, prefix=settings.api_prefix)
-app.include_router(silences.router, prefix=settings.api_prefix)
-app.include_router(notifications.router, prefix=settings.api_prefix)
+# WebSocket — auth is checked inside the handler (WS can't use header deps)
+app.include_router(ws_router, prefix=settings.api_prefix)
+
+# API routes under /api/v1 — all require API key
+api_deps = [Depends(require_api_key)]
+app.include_router(webhooks.router, prefix=settings.api_prefix, dependencies=api_deps)
+app.include_router(alerts.router, prefix=settings.api_prefix, dependencies=api_deps)
+app.include_router(incidents.router, prefix=settings.api_prefix, dependencies=api_deps)
+app.include_router(stats.router, prefix=settings.api_prefix, dependencies=api_deps)
+app.include_router(silences.router, prefix=settings.api_prefix, dependencies=api_deps)
+app.include_router(notifications.router, prefix=settings.api_prefix, dependencies=api_deps)
 
 
 # ─── Startup / Shutdown ──────────────────────────────────

backend/models/__init__.py

@@ -306,7 +306,8 @@ class NotificationChannel(Base):
     )
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     channel_type: Mapped[ChannelType] = mapped_column(
-        Enum(ChannelType), nullable=False
+        Enum(ChannelType, values_callable=lambda x: [e.value for e in x]),
+        nullable=False,
     )
     config: Mapped[dict] = mapped_column(JSONB, default=dict)
     is_active: Mapped[bool] = mapped_column(default=True)
@@ -339,7 +340,9 @@ class NotificationLog(Base):
     )
     event_type: Mapped[str] = mapped_column(String(100), nullable=False)
     status: Mapped[NotificationStatus] = mapped_column(
-        Enum(NotificationStatus), nullable=False, default=NotificationStatus.PENDING
+        Enum(NotificationStatus, values_callable=lambda x: [e.value for e in x]),
+        nullable=False,
+        default=NotificationStatus.PENDING,
     )
     error_message: Mapped[str | None] = mapped_column(Text)
     sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))

backend/services/__init__.py

@@ -57,6 +57,13 @@ async def ingest_alert(
             f"Duplicate alert: {normalized.name} (fingerprint={fingerprint}, "
             f"count={updated.duplicate_count})"
         )
+
+        from backend.api.routes.ws import emit_event
+        await emit_event("alert.updated", {
+            "alert_id": str(updated.id),
+            "duplicate_count": updated.duplicate_count,
+        })
+
         return updated, True
 
     # Step 3b: Check silence windows
@@ -137,6 +144,17 @@ async def ingest_alert(
         f"{f', incident={str(incident.id)[:8]}' if incident else ''})"
     )
 
+    # Emit real-time events
+    from backend.api.routes.ws import emit_event
+
+    await emit_event("alert.created", {"alert_id": str(alert.id), "name": alert.name})
+    if incident and result.event_type == "incident_created":
+        inc_data = {"incident_id": str(incident.id), "title": incident.title}
+        await emit_event("incident.created", inc_data)
+    elif incident and result.event_type == "severity_changed":
+        inc_data = {"incident_id": str(incident.id), "title": incident.title}
+        await emit_event("incident.updated", inc_data)
+
     return alert, False
 
 
@@ -229,6 +247,10 @@ async def acknowledge_alert(
     alert.updated_at = now
     await db.flush()
     await db.refresh(alert)
+
+    from backend.api.routes.ws import emit_event
+    await emit_event("alert.updated", {"alert_id": str(alert.id), "status": "acknowledged"})
+
     return alert
 
 
@@ -251,6 +273,10 @@ async def resolve_alert(
     alert.updated_at = now
     await db.flush()
     await db.refresh(alert)
+
+    from backend.api.routes.ws import emit_event
+    await emit_event("alert.updated", {"alert_id": str(alert.id), "status": "resolved"})
+
     return alert
 
 
@@ -363,6 +389,11 @@ async def acknowledge_incident(
 
     await db.flush()
     await db.refresh(incident)
+
+    from backend.api.routes.ws import emit_event
+    inc_id = str(incident.id)
+    await emit_event("incident.updated", {"incident_id": inc_id, "status": "acknowledged"})
+
     return incident
 
 
@@ -402,6 +433,10 @@ async def resolve_incident(
 
     await db.flush()
     await db.refresh(incident)
+
+    from backend.api.routes.ws import emit_event
+    await emit_event("incident.updated", {"incident_id": str(incident.id), "status": "resolved"})
+
     return incident