error-alert message for the service

.env.example

@@ -83,3 +83,19 @@ PROXY_BASE_URL=https://your-domain.com:8443
 NGINX_CONF_DIR=/etc/nginx/jupyter-locations
 # inotifywait inside the nginx container auto-reloads on file changes — no reload command needed.
 NGINX_RELOAD_CMD=true
+
+# Slack error alerts — posted to your #aistudio-alerts channel.
+# Create one at: https://api.slack.com/apps -> (your app) -> Incoming Webhooks
+# -> Add New Webhook to Workspace -> pick #aistudio-alerts -> copy the URL.
+# Leave blank to disable alerting entirely (every alert call becomes a no-op).
+SLACK_ALERT_WEBHOOK_URL=
+# "Where is this service running" for a SERVICE-level alert (an unhandled API
+# exception, or /health finding Postgres unreachable) — these alerts have no
+# workload, so this is reported instead. Leave blank to auto-detect via the
+# container/host's own hostname resolution, but note that inside an
+# unmodified Docker bridge network this resolves to the CONTAINER's internal
+# IP, not a reachable host IP — set this explicitly in any real deployment.
+SERVICE_HOST_IP= 
+# Minimum seconds between two service-level alerts, so a prolonged Postgres
+# outage (polled by /health every few seconds) doesn't flood the channel.
+SLACK_SERVICE_ALERT_COOLDOWN_SECONDS=300

app/config.py

@@ -77,6 +77,25 @@ class Settings(BaseSettings):
     # Override via environment variable JUPYTER_ASSISTANT_URL.
     JUPYTER_ASSISTANT_URL: str = ""
 
+    # ── Slack Error Alerts ────────────────────────────────────────────────
+    # Incoming Webhook URL for the #aistudio-alerts channel (Slack App ->
+    # "Incoming Webhooks" -> Add New Webhook to Workspace). Leave blank to
+    # disable alerting entirely (every call becomes a no-op) — useful for
+    # local dev where you don't want your own errors paging the channel.
+    SLACK_ALERT_WEBHOOK_URL: str = ""
+    # Explicit override for "where is this service running" in a service-level
+    # alert (main.py's generic exception handler / a degraded /health check).
+    # Leave blank to auto-detect via socket.gethostbyname(gethostname()) — note
+    # that inside an unmodified Docker bridge network this usually resolves to
+    # the CONTAINER's internal IP, not a reachable host IP, so set this
+    # explicitly in production (e.g. to the host's LAN IP or public hostname).
+    SERVICE_HOST_IP: str = ""
+    # Minimum seconds between two service-level alerts. Protects the channel
+    # from being flooded if something (e.g. Postgres) is down for a while and
+    # /health is being polled every few seconds by an external monitor.
+    # Per-process only — not shared across multiple uvicorn workers/replicas.
+    SLACK_SERVICE_ALERT_COOLDOWN_SECONDS: int = 300
+
     class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"

app/main.py

@@ -4,6 +4,7 @@
 from fastapi.exceptions import RequestValidationError
 
 from app.routers import system, ingest, benchmarks, results, jupyter, gpu_specs
+from app.services.slack_notifier import notify_service_error
 
 # The app instance
 app = FastAPI(
@@ -52,6 +53,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
 
 @app.exception_handler(Exception)
 async def generic_exception_handler(request: Request, exc: Exception):
+    # Service-level error -- no workload is necessarily involved, so the
+    # alert reports only when/where, not what (see slack_notifier.py). This
+    # is also the ONLY place an unhandled API exception got logged/surfaced
+    # anywhere server-side before this change -- previously it only went
+    # back to the client in the JSON response.
+    await notify_service_error()
     return JSONResponse(
         status_code=500,
         content={

app/routers/system.py

@@ -5,6 +5,7 @@
 from app.catalog import _DEFAULT_CONFIG, _MODEL_CONFIGS, _MODEL_INFO
 from app.database import get_db
 from app.models.workload_type import WorkloadType
+from app.services.slack_notifier import notify_service_error
 
 router = APIRouter(tags=["System"])
 
@@ -21,7 +22,12 @@ async def health_check(db: AsyncSession = Depends(get_db)):
         db_status = "ok"
     except Exception as e:
         db_status = f"unreachable: {str(e)}"
-
+        # Service is down (or erroring while trying to come back up) --
+        # alert with just when + where; notify_service_error() rate-limits
+        # itself so a monitor polling /health every few seconds during a
+        # prolonged outage doesn't flood the channel with one post each poll.
+        await notify_service_error()
+
     return {
         "status": "healthy" if db_status == "ok" else "degraded",
         "database": db_status,

app/services/slack_notifier.py

@@ -0,0 +1,154 @@
+"""
+Slack error alerting -- best-effort, fire-and-forget notifications to the
+#aistudio-alerts channel.
+
+Two alert shapes, matching exactly what the RCA agent needs to start digging
+in, and nothing more (no error text is sent -- the agent recovers that
+itself, from workload_events.message / workloads.error_message / task_logs,
+via the tools it already has):
+
+  1. Workload failure  -> Workload ID, Timestamp, Node IP
+     Fired from app.worker._fail_workload() -- the single choke point every
+     Celery task (validate_node / install_dependencies / execute_benchmark /
+     launch_jupyter) already routes through on any failure, so one hook here
+     covers every "error in benchmarking / starting the benchmark / etc"
+     scenario without instrumenting each task individually.
+
+  2. Service error     -> Timestamp, Service IP
+     Fired when the API process itself is in trouble: an unhandled exception
+     (app.main's generic_exception_handler) or a degraded /health check
+     (Postgres unreachable, i.e. "down, or erroring while trying to come back
+     up"). No workload is involved here, so there's no workload ID to
+     report -- just where the *service* is running.
+
+Uses httpx (already a project dependency) against a Slack Incoming Webhook
+URL rather than the Slack SDK/bot token: an Incoming Webhook is a single
+POST of {"text": "..."} to a per-channel URL -- no bot token, no signing
+secret, no app-level OAuth needed on THIS side (that machinery only matters
+for the side that has to *read* Slack, which is the RCA agent, not here).
+
+Every public function is wrapped so a Slack outage or missing/blank webhook
+URL can NEVER raise into the caller's real code path -- a Celery task
+failing to post an alert must still finish marking the workload FAILED, and
+a FastAPI request failing to post an alert must still return its error
+response to the client.
+"""
+
+from __future__ import annotations
+
+import logging
+import socket
+from datetime import datetime, timedelta, timezone
+
+import httpx
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+_ISO_FMT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+def _resolve_service_ip() -> str:
+    """settings.SERVICE_HOST_IP if set, else a best-effort auto-detect.
+
+    NOTE: inside an unmodified Docker bridge network, gethostbyname(hostname())
+    resolves to the CONTAINER's internal IP, not a reachable host IP -- set
+    SERVICE_HOST_IP explicitly in any deployment where that distinction matters
+    (which is effectively every real deployment).
+    """
+    if settings.SERVICE_HOST_IP:
+        return settings.SERVICE_HOST_IP
+    try:
+        return socket.gethostbyname(socket.gethostname())
+    except OSError:
+        return "unknown"
+
+
+def _fmt_ts(ts: datetime) -> str:
+    return ts.astimezone(timezone.utc).strftime(_ISO_FMT)
+
+
+def _workload_failure_text(workload_id: str, node_ip: str | None, occurred_at: datetime) -> str:
+    return (
+        "🚨 *Workload Failed*\n"
+        f"Workload: `{workload_id}`\n"
+        f"Time: `{_fmt_ts(occurred_at)}`\n"
+        f"Node: `{node_ip or 'unknown'}`"
+    )
+
+
+def _service_error_text(occurred_at: datetime, service_ip: str) -> str:
+    return (
+        "🔴 *aistudio-server error*\n"
+        f"Time: `{_fmt_ts(occurred_at)}`\n"
+        f"Service: `{service_ip}`"
+    )
+
+
+def _send_sync(text: str) -> None:
+    """Post from a synchronous context (the Celery worker)."""
+    if not settings.SLACK_ALERT_WEBHOOK_URL:
+        return  # alerting disabled -- no-op, not an error
+    try:
+        resp = httpx.post(settings.SLACK_ALERT_WEBHOOK_URL, json={"text": text}, timeout=5.0)
+        resp.raise_for_status()
+    except Exception as exc:  # noqa: BLE001 -- alerting must never break the caller
+        logger.warning("slack_notifier: failed to post alert: %s", exc)
+
+
+async def _send_async(text: str) -> None:
+    """Post from an async context (FastAPI request handlers)."""
+    if not settings.SLACK_ALERT_WEBHOOK_URL:
+        return
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            resp = await client.post(settings.SLACK_ALERT_WEBHOOK_URL, json={"text": text})
+            resp.raise_for_status()
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("slack_notifier: failed to post alert: %s", exc)
+
+
+def notify_workload_failure(
+    workload_id: str,
+    node_ip: str | None,
+    occurred_at: datetime | None = None,
+) -> None:
+    """Fire a workload-failure alert. Call from a SYNCHRONOUS context (Celery tasks).
+
+    Args:
+        workload_id: The human-readable workload id (e.g. 'wl-20260806-oom01').
+        node_ip: The GPU node the workload was running on, if known.
+        occurred_at: When the failure was detected. Defaults to now() -- pass
+            an explicit value if the caller already captured a more precise
+            moment (e.g. right when the exception was caught).
+    """
+    occurred_at = occurred_at or datetime.now(timezone.utc)
+    _send_sync(_workload_failure_text(workload_id, node_ip, occurred_at))
+
+
+# Service-level alert cooldown -- per-process only (see
+# SLACK_SERVICE_ALERT_COOLDOWN_SECONDS' docstring in config.py for why).
+_last_service_alert_at: datetime | None = None
+
+
+def _service_cooldown_active(now: datetime) -> bool:
+    global _last_service_alert_at
+    cooldown = timedelta(seconds=settings.SLACK_SERVICE_ALERT_COOLDOWN_SECONDS)
+    if _last_service_alert_at is not None and (now - _last_service_alert_at) < cooldown:
+        return True
+    _last_service_alert_at = now
+    return False
+
+
+async def notify_service_error(occurred_at: datetime | None = None) -> None:
+    """Fire a service-level alert. Call from an ASYNC context (FastAPI).
+
+    Rate-limited by SLACK_SERVICE_ALERT_COOLDOWN_SECONDS so a Postgres outage
+    being polled by /health every few seconds doesn't flood the channel with
+    one alert per poll.
+    """
+    now = occurred_at or datetime.now(timezone.utc)
+    if _service_cooldown_active(now):
+        return
+    await _send_async(_service_error_text(now, _resolve_service_ip()))

app/worker.py

@@ -21,6 +21,7 @@
 from app.services.manifest_builder import ManifestBuilder
 from app.services.nginx_proxy import write_jupyter_config, proxy_url as nginx_proxy_url, jupyter_base_path
 from app.services.ssh_executor import SSHExecutor
+from app.services.slack_notifier import notify_workload_failure
 from app.services.state_machine import transition_workload_state
 
 logger = logging.getLogger(__name__)
@@ -89,30 +90,45 @@ def _fail_workload(workload_id, trigger, error):
     For benchmark workloads (not Jupyter) also writes a BenchmarkResult row with
     status='failed' so the run appears in the leaderboard API after a page refresh
     instead of disappearing when the in-memory stream store is cleared.
+
+    This is the single choke point every Celery task (validate_node /
+    install_dependencies / execute_benchmark / launch_jupyter) routes through
+    on any failure -- so it's also the one place we fire the Slack alert,
+    rather than instrumenting each task's except block individually.
     """
+    occurred_at = datetime.now(timezone.utc)
+    node_ip = None
     try:
         with SyncSessionLocal() as db:
             transition_workload_state(
                 db, workload_id, WorkloadState.FAILED,
                 trigger=trigger, message=error,
             )
+            workload = db.query(Workload).filter(Workload.workload_id == workload_id).first()
+
+            # Node lookup -- always attempted (benchmark AND jupyter workloads)
+            # so the Slack alert can report where the workload was running.
+            # A workload can fail before any node was ever assigned (e.g. "No
+            # nodes available"), so node_ip may legitimately stay None.
+            if workload:
+                node = db.query(Node).filter(Node.workload_id == workload.id).first()
+                node_ip = node.machine_ip if node else None
+
             # Write a minimal BenchmarkResult so GET /api/v1/benchmarks returns
             # the failed run. Skip Jupyter workloads — they have no benchmark metrics.
-            workload = db.query(Workload).filter(Workload.workload_id == workload_id).first()
             if workload and (workload.workload_config or {}).get("workload_type") != "jupyter":
                 existing = db.query(BenchmarkResult).filter(
                     BenchmarkResult.run_id == workload_id
                 ).first()
                 now = datetime.now(timezone.utc)
                 if not existing:
-                    node = db.query(Node).filter(Node.workload_id == workload.id).first()
                     db.add(BenchmarkResult(
                         run_id=workload_id,
                         sub_run_index=0,
                         workload_type="llm",
                         model_name=(workload.model_name or "").lower(),
                         pipeline_version="vllm-openai:v0.14.1",
-                        node_ips=[node.machine_ip] if node else [],
+                        node_ips=[node_ip] if node_ip else [],
                         gpu_type="",
                         gpu_count=0,
                         gpu_model="",
@@ -145,6 +161,12 @@ def _fail_workload(workload_id, trigger, error):
             workload_id, trigger,
             exc_info=True,
         )
+        return  # DB write itself failed -- state is unreliable, don't alert on it
+
+    # Fired outside the try/except above: the FAILED state is durably written
+    # by this point, and notify_workload_failure() never raises (Slack being
+    # down must not affect this function's real job of failing the workload).
+    notify_workload_failure(workload_id, node_ip, occurred_at)
 
 
 def _fetch_workload_and_nodes(db, workload_id: str):