diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bf9ab5..e00269e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,13 +83,13 @@ jobs: as_bot 'SELECT count(*) FROM "InstallGrant"' as_bot 'SELECT count(*) FROM "StatusChannel"' as_bot "INSERT INTO \"InstanceEvent\" (id, source, level, title, details) VALUES ('ci', 'bot', 'info', 'ci', '{}')" - for table in PanelSession PanelAudit PanelRoleBinding SuperAdminAudit InstanceConfig WaitlistEntry Coupon InstanceEvent; do + for table in PanelSession PanelAudit PanelRoleBinding SuperAdminAudit InstanceConfig WaitlistEntry Coupon InstanceEvent Incident; do if as_bot "SELECT count(*) FROM \"$table\"" 2>/dev/null; then echo "sentinel_bot can read $table" >&2; exit 1; fi done for table in InstallBlock InstallGrant StatusChannel; do if as_bot "DELETE FROM \"$table\"" 2>/dev/null; then echo "sentinel_bot can modify $table" >&2; exit 1; fi done - for table in GuildSettings LogRoute LogEvent MessageSnapshot PanelAudit PanelSession PanelRoleBinding InstallBlock SuperAdminAudit InstanceConfig StatusChannel WaitlistEntry InstallGrant Coupon InstanceEvent; do + for table in GuildSettings LogRoute LogEvent MessageSnapshot PanelAudit PanelSession PanelRoleBinding InstallBlock SuperAdminAudit InstanceConfig StatusChannel WaitlistEntry InstallGrant Coupon InstanceEvent Incident; do as_api "SELECT count(*) FROM \"$table\"" done if as_api 'CREATE TABLE ci_should_fail (id int)' 2>/dev/null; then echo 'sentinel_api can create tables' >&2; exit 1; fi diff --git a/README.md b/README.md index 8083b66..59c3230 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ The full list of limits lives in [`packages/shared/src/plans.ts`](packages/share - public contact details shown on the pricing page; - **Discord subscriptions**: the SKU ID of each plan and billing period, the last sync and a manual "sync now"; - the **status channel**: one server and channel, chosen among those the bot can see, that receives bot startup, errors, warnings, reached quotas and servers added, removed or rejected — these notices are no longer posted in individual servers; -- **monitoring**: the bot watchdog (checked every 30 seconds by the API) with *down* and *restored* alarms posted to a Discord webhook, which works even when the bot is stopped, and the feed of instance events (startup, errors, servers added, removed or rejected, quotas), kept for 90 days; +- **monitoring and alerts**: every alert has a category (availability, security, errors, log delivery, subscriptions, installations, usage) and a severity. Each category goes to the status channel, another channel chosen from the bot's servers, the daily digest or the log only; identical alerts within 10 minutes are merged, critical ones mention the owner and ignore the optional quiet hours. Lasting problems open **incidents** (bot down, Discord subscription sync stopped, log queue backlog, many failed deliveries) with reminders after 15 minutes, 1 hour and every 6 hours until acknowledged. *Bot down* is recorded by the API watchdog and reported by an external monitor on `/backend/health/bot` (optionally also to a Discord webhook); events are kept for 90 days; - **logs**: the latest log lines of the bot and the API since their last restart, redacted and kept in memory only; Docker log files are capped at 5 × 10 MB per service; - installation blacklist and an audit log of every action, including automatic plan expiries. diff --git a/apps/api/src/alert-rules.ts b/apps/api/src/alert-rules.ts new file mode 100644 index 0000000..fe1006d --- /dev/null +++ b/apps/api/src/alert-rules.ts @@ -0,0 +1,154 @@ +import { ALERT_CATEGORIES, type AlertCategory, type AlertSeverity } from '@sentinel/shared'; + +/** + * Pure alert routing rules: where each category goes, quiet hours, the daily + * digest, grouping of repeated alerts and incident reminders. + */ + +export type RouteMode = 'status' | 'channel' | 'digest' | 'log'; +export type AlertRoute = { mode: RouteMode; guildId: string | null; channelId: string | null; mention: boolean }; +export type AlertTarget = { guildId: string; channelId: string }; +export type AlertSettings = { + routes: Record; + quietHours: { enabled: boolean; start: string; end: string }; + digest: { time: string; mode: 'status' | 'channel'; guildId: string | null; channelId: string | null }; + timezone: string; +}; + +const route = (mode: RouteMode, mention = false): AlertRoute => ({ mode, guildId: null, channelId: null, mention }); + +export const DEFAULT_ALERT_SETTINGS: AlertSettings = { + routes: { + availability: route('status', true), + security: route('status', true), + errors: route('status'), + delivery: route('status'), + billing: route('status'), + installations: route('digest'), + usage: route('digest') + }, + quietHours: { enabled: false, start: '23:00', end: '08:00' }, + digest: { time: '09:00', mode: 'status', guildId: null, channelId: null }, + timezone: 'Europe/Rome' +}; + +const SNOWFLAKE = /^\d{17,20}$/; +const TIME = /^([01]\d|2[0-3]):[0-5]\d$/; +const MODES = new Set(['status', 'channel', 'digest', 'log']); + +export const isTimeZone = (value: unknown): value is string => { + if (typeof value !== 'string' || !value) return false; + try { + new Intl.DateTimeFormat('en-GB', { timeZone: value }); + return true; + } catch { + return false; + } +}; + +const obj = (value: unknown) => (value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}); +const id = (value: unknown) => (typeof value === 'string' && SNOWFLAKE.test(value) ? value : null); +const time = (value: unknown, fallback: string) => (typeof value === 'string' && TIME.test(value) ? value : fallback); + +/** Stored settings merged over the defaults; invalid values fall back. */ +export function parseAlertSettings(value: unknown): AlertSettings { + const input = obj(value); + const routesIn = obj(input.routes); + const routes = {} as Record; + for (const category of ALERT_CATEGORIES) { + const base = DEFAULT_ALERT_SETTINGS.routes[category]; + const raw = obj(routesIn[category]); + const mode = MODES.has(raw.mode as RouteMode) ? raw.mode as RouteMode : base.mode; + routes[category] = { + mode, + guildId: mode === 'channel' ? id(raw.guildId) : null, + channelId: mode === 'channel' ? id(raw.channelId) : null, + mention: typeof raw.mention === 'boolean' ? raw.mention : base.mention + }; + } + const quiet = obj(input.quietHours); + const digest = obj(input.digest); + const digestMode = digest.mode === 'channel' ? 'channel' : 'status'; + return { + routes, + quietHours: { + enabled: quiet.enabled === true, + start: time(quiet.start, DEFAULT_ALERT_SETTINGS.quietHours.start), + end: time(quiet.end, DEFAULT_ALERT_SETTINGS.quietHours.end) + }, + digest: { + time: time(digest.time, DEFAULT_ALERT_SETTINGS.digest.time), + mode: digestMode, + guildId: digestMode === 'channel' ? id(digest.guildId) : null, + channelId: digestMode === 'channel' ? id(digest.channelId) : null + }, + timezone: isTimeZone(input.timezone) ? input.timezone : DEFAULT_ALERT_SETTINGS.timezone + }; +} + +function localParts(now: Date, timeZone: string) { + const parts = Object.fromEntries(new Intl.DateTimeFormat('en-GB', { + timeZone, year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hourCycle: 'h23' + }).formatToParts(now).map((part) => [part.type, part.value])); + return { date: `${parts.year}-${parts.month}-${parts.day}`, minutes: Number(parts.hour) * 60 + Number(parts.minute) }; +} + +const minutesOf = (value: string) => Number(value.slice(0, 2)) * 60 + Number(value.slice(3, 5)); + +/** Quiet hours may cross midnight (e.g. 23:00–08:00). */ +export function inQuietHours(settings: AlertSettings, now: Date) { + if (!settings.quietHours.enabled) return false; + const start = minutesOf(settings.quietHours.start); + const end = minutesOf(settings.quietHours.end); + if (start === end) return false; + const { minutes } = localParts(now, settings.timezone); + return start < end ? minutes >= start && minutes < end : minutes >= start || minutes < end; +} + +/** Once per local day, at or after the digest time. */ +export function digestDue(settings: AlertSettings, now: Date, lastDigestAt: Date | null) { + const today = localParts(now, settings.timezone); + if (today.minutes < minutesOf(settings.digest.time)) return false; + return !lastDigestAt || localParts(lastDigestAt, settings.timezone).date !== today.date; +} + +/** Where a route delivers; null means it cannot be delivered right now. */ +export function routeTarget(target: Pick, statusChannel: AlertTarget | null): AlertTarget | null { + if (target.mode === 'status') return statusChannel; + if (target.mode === 'channel' && target.guildId && target.channelId) return { guildId: target.guildId, channelId: target.channelId }; + return null; +} + +/** Repeated alerts with the same key are merged and held back for a window. */ +export const GROUP_WINDOW_MS = 10 * 60_000; + +export function groupForDelivery(items: T[], lastSent: Map, now: number, windowMs = GROUP_WINDOW_MS) { + const groups = new Map(); + for (const item of items) { + const last = lastSent.get(item.key); + if (last !== undefined && now - last < windowMs) continue; + groups.set(item.key, [...(groups.get(item.key) ?? []), item]); + } + return [...groups.entries()].map(([key, members]) => ({ key, members })); +} + +/** Reminders for an open, unacknowledged incident: after 15 min, 1 h, then every 6 h. */ +export function reminderDue(incident: { + severity: string; openedAt: Date; resolvedAt: Date | null; acknowledgedAt: Date | null; lastRemindedAt: Date | null; reminders: number; +}, now: Date) { + if (incident.resolvedAt || incident.acknowledgedAt || incident.severity === 'info') return false; + const sinceOpen = now.getTime() - incident.openedAt.getTime(); + if (incident.reminders === 0) return sinceOpen >= 15 * 60_000; + if (incident.reminders === 1) return sinceOpen >= 60 * 60_000; + return !incident.lastRemindedAt || now.getTime() - incident.lastRemindedAt.getTime() >= 6 * 60 * 60_000; +} + +export function formatDuration(ms: number) { + const minutes = Math.max(1, Math.round(ms / 60_000)); + if (minutes < 60) return `${minutes} min`; + const hours = Math.floor(minutes / 60); + if (hours < 48) return `${hours} h ${minutes % 60} min`; + return `${Math.floor(hours / 24)} giorni`; +} + +export const SEVERITY_COLOR: Record = { critical: 0xf2555a, warning: 0xf5a524, info: 0xa78bfa, ok: 0x3ccf8e }; diff --git a/apps/api/src/alerting.ts b/apps/api/src/alerting.ts new file mode 100644 index 0000000..9c9ebc3 --- /dev/null +++ b/apps/api/src/alerting.ts @@ -0,0 +1,351 @@ +import type { FastifyInstance } from 'fastify'; +import { prisma } from '@sentinel/db'; +import { ALERT_CATEGORIES, isAlertCategory, type AlertCategory, type AlertSeverity } from '@sentinel/shared'; +import { requireSuperAdmin } from './auth.js'; +import { config } from './config.js'; +import { deliverAlert, getGuildResources } from './discord.js'; +import { findForeignReference } from './helpers.js'; +import { decryptText, protectJson, unprotectJson } from './security.js'; +import { superAdminAudit } from './super-audit.js'; +import { + SEVERITY_COLOR, + digestDue, + formatDuration, + groupForDelivery, + inQuietHours, + parseAlertSettings, + reminderDue, + routeTarget, + type AlertSettings, + type AlertTarget +} from './alert-rules.js'; + +/** + * Instance alert routing. The bot and the API record alerts as InstanceEvent + * rows; this router sends them where the super console says (status channel, + * another channel, daily digest, log only), merges repeats, defers + * non-critical alerts during quiet hours and reminds open incidents. Discord + * messages are posted by the bot, so an outage of the bot itself is reported + * by the external monitor (and the optional webhook), not here. + */ + +type Logger = { info: (obj: object, msg: string) => void; warn: (obj: object, msg: string) => void }; +type Level = 'info' | 'ok' | 'warn' | 'error'; +type Details = { description?: string | null; fields?: Array<{ name: string; value: string }> }; + +const ROUTER_INTERVAL_MS = 20_000; +const DELIVERY_CHECK_MS = 2 * 60_000; +const PENDING_MAX_AGE_MS = 24 * 60 * 60_000; + +export const CATEGORY_LABELS: Record = { + availability: 'Disponibilità', + security: 'Sicurezza', + errors: 'Errori', + delivery: 'Consegna dei log', + billing: 'Abbonamenti', + installations: 'Installazioni', + usage: 'Utilizzo' +}; + +const LEVEL_COLOR: Record = { ok: SEVERITY_COLOR.ok, info: SEVERITY_COLOR.info, warn: SEVERITY_COLOR.warning, error: SEVERITY_COLOR.critical }; +const levelFor = (severity: AlertSeverity): Level => severity === 'critical' ? 'error' : severity === 'warning' ? 'warn' : 'info'; + +const readDetails = (value: unknown): Details => { + try { + return unprotectJson(value) as Details; + } catch { + return {}; + } +}; + +/* ------------------------------------------------------------- recording */ + +export async function recordAlert(input: { + category: AlertCategory; severity: AlertSeverity; title: string; description?: string | null; + fields?: Array<{ name: string; value: string }>; level?: Level; logOnly?: boolean; +}) { + await prisma.instanceEvent.create({ + data: { + source: 'api', + level: input.level ?? levelFor(input.severity), + category: input.category, + severity: input.severity, + title: input.title.slice(0, 256), + details: protectJson({ description: input.description ?? null, fields: input.fields ?? [] }), + deliveryState: input.logOnly ? 'log' : 'pending' + } + }); +} + +/** Opens an incident once per key; repeated calls while it is open do nothing. */ +export async function openIncident(key: string, input: { + category: AlertCategory; severity: AlertSeverity; title: string; description: string; logOnly?: boolean; +}) { + const open = await prisma.incident.findFirst({ where: { key, resolvedAt: null } }); + if (open) return false; + await prisma.incident.create({ data: { key, category: input.category, severity: input.severity, title: input.title, description: input.description } }); + await recordAlert({ category: input.category, severity: input.severity, title: input.title, description: input.description, logOnly: input.logOnly }); + return true; +} + +/** Resolves the open incident of a key and announces it with its duration. */ +export async function resolveIncident(key: string, message: { title: string; description: (duration: string) => string }) { + const open = await prisma.incident.findFirst({ where: { key, resolvedAt: null } }); + if (!open) return null; + const now = new Date(); + await prisma.incident.update({ where: { id: open.id }, data: { resolvedAt: now } }); + const duration = formatDuration(now.getTime() - open.openedAt.getTime()); + await recordAlert({ category: isAlertCategory(open.category) ? open.category : 'errors', severity: 'info', level: 'ok', title: message.title, description: message.description(duration) }); + return duration; +} + +/* --------------------------------------------------------------- webhook */ + +const WEBHOOK_COLORS: Record = LEVEL_COLOR; + +export async function alertWebhookUrl() { + const row = await prisma.instanceConfig.findUnique({ where: { id: 1 }, select: { alertWebhookUrl: true } }); + return row?.alertWebhookUrl ? decryptText(row.alertWebhookUrl) : null; +} + +/** Optional fallback for "bot down": a webhook works while the bot is stopped. */ +export async function postAlertWebhook(url: string, level: Level, title: string, description: string) { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + username: 'Sentinel monitor', + allowed_mentions: { parse: [] }, + embeds: [{ title, description, color: WEBHOOK_COLORS[level], timestamp: new Date().toISOString(), footer: { text: 'Sentinel · monitoraggio' } }] + }), + signal: AbortSignal.timeout(8_000) + }); + if (!response.ok) throw new Error(`Webhook responded ${response.status}`); +} + +/* ---------------------------------------------------------------- router */ + +async function loadSettings() { + const row = await prisma.instanceConfig.upsert({ where: { id: 1 }, update: {}, create: { id: 1 }, select: { alertSettings: true, lastDigestAt: true } }); + return { settings: parseAlertSettings(row.alertSettings), lastDigestAt: row.lastDigestAt }; +} + +async function statusTarget(): Promise { + const row = await prisma.statusChannel.findUnique({ where: { id: 1 }, select: { guildId: true, channelId: true } }); + return row ? { guildId: row.guildId, channelId: row.channelId } : null; +} + +const mentionFor = (mention: boolean, severity: string) => (mention && severity === 'critical' && config.superAdminUserId ? config.superAdminUserId : null); + +const lastSent = new Map(); +let routing = false; + +export async function runAlertRouter(log: Logger) { + if (routing) return; + routing = true; + try { + const now = new Date(); + const [{ settings, lastDigestAt }, status] = await Promise.all([loadSettings(), statusTarget()]); + const quiet = inQuietHours(settings, now); + const pending = await prisma.instanceEvent.findMany({ where: { deliveryState: 'pending' }, orderBy: { createdAt: 'asc' }, take: 300 }); + + const toLog: string[] = []; + const toDigest: string[] = []; + const expired: string[] = []; + const sendable: Array<{ key: string; event: (typeof pending)[number]; target: AlertTarget; mention: string | null; category: AlertCategory }> = []; + for (const event of pending) { + if (now.getTime() - event.createdAt.getTime() > PENDING_MAX_AGE_MS) { expired.push(event.id); continue; } + const category: AlertCategory = isAlertCategory(event.category) ? event.category : 'errors'; + const route = settings.routes[category]; + if (route.mode === 'log') { toLog.push(event.id); continue; } + if (route.mode === 'digest') { toDigest.push(event.id); continue; } + const target = routeTarget(route, status); + if (!target) { toLog.push(event.id); continue; } + // Quiet hours: non-critical alerts wait and leave together in the morning. + if (quiet && event.severity !== 'critical') continue; + sendable.push({ key: `${target.guildId}:${target.channelId}:${event.title}`, event, target, mention: mentionFor(route.mention, event.severity), category }); + } + await Promise.all([ + toLog.length ? prisma.instanceEvent.updateMany({ where: { id: { in: toLog } }, data: { deliveryState: 'log' } }) : null, + toDigest.length ? prisma.instanceEvent.updateMany({ where: { id: { in: toDigest } }, data: { deliveryState: 'digest' } }) : null, + expired.length ? prisma.instanceEvent.updateMany({ where: { id: { in: expired } }, data: { deliveryState: 'failed' } }) : null + ]); + + for (const group of groupForDelivery(sendable, lastSent, now.getTime())) { + const first = group.members[0]!; + const latest = group.members[group.members.length - 1]!; + const details = readDetails(latest.event.details); + const count = group.members.length; + const since = first.event.createdAt.toLocaleTimeString('it-IT', { hour: '2-digit', minute: '2-digit', timeZone: settings.timezone }); + try { + await deliverAlert({ + ...latest.target, + title: count > 1 ? `${latest.event.title} ×${count}` : latest.event.title, + description: [details.description, count > 1 ? `${count} avvisi uguali dalle ${since}; qui sopra il più recente.` : null].filter(Boolean).join('\n\n'), + fields: details.fields ?? [], + color: LEVEL_COLOR[(latest.event.level as Level)] ?? LEVEL_COLOR.info, + footer: `Sentinel · ${CATEGORY_LABELS[latest.category]}`, + mentionUserId: group.members.some((member) => member.mention) ? latest.mention ?? first.mention : null + }); + lastSent.set(group.key, now.getTime()); + await prisma.instanceEvent.updateMany({ where: { id: { in: group.members.map((member) => member.event.id) } }, data: { deliveryState: 'sent', deliveredAt: now } }); + } catch (error) { + // Stays pending: retried at the next run (e.g. while the bot restarts). + log.warn({ err: error }, 'Alert delivery failed'); + } + } + + if (digestDue(settings, now, lastDigestAt)) await sendDigest(settings, status, now, log); + await sendReminders(settings, status, now, quiet, log); + } finally { + routing = false; + } +} + +async function sendDigest(settings: AlertSettings, status: AlertTarget | null, now: Date, log: Logger) { + const target = settings.digest.mode === 'channel' ? routeTarget({ mode: 'channel', guildId: settings.digest.guildId, channelId: settings.digest.channelId }, status) : status; + const events = await prisma.instanceEvent.findMany({ where: { deliveryState: 'digest' }, orderBy: { createdAt: 'asc' }, take: 2_000 }); + const done = () => prisma.instanceConfig.update({ where: { id: 1 }, data: { lastDigestAt: now } }); + if (!events.length) { await done(); return; } + if (!target) { + await prisma.instanceEvent.updateMany({ where: { id: { in: events.map((event) => event.id) } }, data: { deliveryState: 'log' } }); + await done(); + return; + } + + const byCategory = new Map>(); + for (const event of events) { + const category: AlertCategory = isAlertCategory(event.category) ? event.category : 'errors'; + const titles = byCategory.get(category) ?? new Map(); + titles.set(event.title, (titles.get(event.title) ?? 0) + 1); + byCategory.set(category, titles); + } + const fields = ALERT_CATEGORIES.filter((category) => byCategory.has(category)).map((category) => ({ + name: CATEGORY_LABELS[category], + value: [...byCategory.get(category)!.entries()].sort((a, b) => b[1] - a[1]).map(([title, count]) => `• ${title}: **${count}**`).join('\n').slice(0, 1024) + })); + try { + await deliverAlert({ + ...target, + title: 'Riepilogo giornaliero', + description: `${events.length} avvisi informativi dall’ultimo riepilogo. I dettagli sono nella super console → Monitoraggio.`, + fields, + color: SEVERITY_COLOR.info, + footer: 'Sentinel · riepilogo', + mentionUserId: null + }); + await prisma.instanceEvent.updateMany({ where: { id: { in: events.map((event) => event.id) } }, data: { deliveryState: 'digested', deliveredAt: now } }); + await done(); + } catch (error) { + log.warn({ err: error }, 'Daily digest delivery failed'); + } +} + +async function sendReminders(settings: AlertSettings, status: AlertTarget | null, now: Date, quiet: boolean, log: Logger) { + const open = await prisma.incident.findMany({ where: { resolvedAt: null, acknowledgedAt: null } }); + for (const incident of open) { + if (!reminderDue(incident, now)) continue; + if (quiet && incident.severity !== 'critical') continue; + const description = `Ancora aperto da ${formatDuration(now.getTime() - incident.openedAt.getTime())}. Segnalo come preso in carico nella super console → Monitoraggio per non ricevere altri promemoria.`; + try { + if (incident.key === 'bot.down') { + // The bot cannot post while it is down: only the webhook can remind. + const url = await alertWebhookUrl().catch(() => null); + if (url) await postAlertWebhook(url, 'error', `Promemoria: ${incident.title}`, description); + } else { + const category: AlertCategory = isAlertCategory(incident.category) ? incident.category : 'errors'; + const route = settings.routes[category]; + const target = route.mode === 'status' || route.mode === 'channel' ? routeTarget(route, status) : status; + if (target) { + await deliverAlert({ + ...target, + title: `Promemoria: ${incident.title}`, + description: [incident.description, description].filter(Boolean).join('\n\n'), + fields: [], + color: incident.severity === 'critical' ? SEVERITY_COLOR.critical : SEVERITY_COLOR.warning, + footer: `Sentinel · ${CATEGORY_LABELS[category]}`, + mentionUserId: mentionFor(route.mention, incident.severity) + }); + } + } + await prisma.incident.update({ where: { id: incident.id }, data: { reminders: { increment: 1 }, lastRemindedAt: now } }); + } catch (error) { + log.warn({ err: error, incident: incident.key }, 'Incident reminder failed'); + } + } +} + +/* ------------------------------------------------------- delivery health */ + +/** Opens or resolves the incidents about log delivery across all servers. */ +async function checkDelivery() { + const now = Date.now(); + const [backlog, failures] = await Promise.all([ + prisma.logEvent.count({ where: { dispatchState: 'PENDING', createdAt: { lt: new Date(now - 10 * 60_000), gt: new Date(now - PENDING_MAX_AGE_MS) } } }), + prisma.logEvent.count({ where: { dispatchState: 'FAILED', createdAt: { gt: new Date(now - 15 * 60_000) } } }) + ]); + if (backlog >= 100) { + await openIncident('delivery.backlog', { category: 'delivery', severity: 'warning', title: 'Coda dei log in accumulo', description: `${backlog} log attendono la consegna da più di 10 minuti: il dispatcher del bot è fermo o rallentato.` }); + } else if (backlog <= 20) { + await resolveIncident('delivery.backlog', { title: 'Coda dei log smaltita', description: (duration) => `La consegna dei log è tornata regolare dopo ${duration}.` }); + } + if (failures >= 25) { + await openIncident('delivery.failures', { category: 'delivery', severity: 'warning', title: 'Molti invii dei log falliti', description: `${failures} log rifiutati da Discord negli ultimi 15 minuti.` }); + } else if (failures < 5) { + await resolveIncident('delivery.failures', { title: 'Invii dei log tornati regolari', description: (duration) => `Gli invii falliti sono rientrati dopo ${duration}.` }); + } +} + +export function startAlerting(log: Logger) { + setInterval(() => void runAlertRouter(log).catch((error) => log.warn({ err: error }, 'Alert router failed')), ROUTER_INTERVAL_MS).unref(); + setInterval(() => void checkDelivery().catch((error) => log.warn({ err: error }, 'Delivery check failed')), DELIVERY_CHECK_MS).unref(); +} + +/* ---------------------------------------------------------------- routes */ + +export async function registerAlertRoutes(app: FastifyInstance) { + app.get('/api/super/alerts', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const [{ settings }, open, recent] = await Promise.all([ + loadSettings(), + prisma.incident.findMany({ where: { resolvedAt: null }, orderBy: { openedAt: 'desc' } }), + prisma.incident.findMany({ where: { resolvedAt: { not: null } }, orderBy: { openedAt: 'desc' }, take: 30 }) + ]); + return { settings, incidents: { open, recent }, mentionAvailable: Boolean(config.superAdminUserId) }; + }); + + app.put('/api/super/alerts/settings', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const settings = parseAlertSettings(request.body); + // Every chosen channel must belong to its server and accept messages. + const targets = [ + ...Object.values(settings.routes).filter((route) => route.mode === 'channel'), + ...(settings.digest.mode === 'channel' ? [settings.digest] : []) + ]; + if (targets.some((target) => !target.guildId || !target.channelId)) return reply.code(400).send({ error: 'CHANNEL_REQUIRED' }); + try { + for (const target of targets) { + const resources = await getGuildResources(target.guildId!); + if (findForeignReference(resources, { sendableChannelIds: [target.channelId!] })) return reply.code(400).send({ error: 'UNKNOWN_GUILD_REFERENCE' }); + } + } catch { + return reply.code(502).send({ error: 'DISCORD_RESOURCES_FAILED' }); + } + await prisma.instanceConfig.upsert({ where: { id: 1 }, update: { alertSettings: settings }, create: { id: 1, alertSettings: settings } }); + await superAdminAudit(request, session, 'alerts.settings', null, null, settings); + return { settings }; + }); + + app.post('/api/super/incidents/:id/ack', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const { id } = request.params as { id: string }; + const incident = await prisma.incident.findUnique({ where: { id } }); + if (!incident || incident.resolvedAt) return reply.code(404).send({ error: 'NOT_FOUND' }); + await prisma.incident.update({ where: { id }, data: { acknowledgedAt: new Date(), acknowledgedBy: session.username } }); + await superAdminAudit(request, session, 'incident.ack', 'INCIDENT', incident.key); + return { ok: true }; + }); +} diff --git a/apps/api/src/discord-billing.ts b/apps/api/src/discord-billing.ts index 8b1c45b..b467be4 100644 --- a/apps/api/src/discord-billing.ts +++ b/apps/api/src/discord-billing.ts @@ -2,6 +2,8 @@ import { prisma } from '@sentinel/db'; import { getEntitlements, type Entitlement } from './discord.js'; import { decidePlan, parseSkus, plansFromEntitlements, skuIndex } from './discord-plans.js'; import { applyPlanLimits, systemAudit } from './plan-service.js'; +import { openIncident, recordAlert, resolveIncident } from './alerting.js'; +import { PLANS } from '@sentinel/shared'; export { PAID_TIERS, SKU_PERIODS, parseSkus, storeLinks } from './discord-plans.js'; @@ -9,6 +11,9 @@ type Logger = { warn: (obj: object, msg: string) => void; error: (obj: object, m export type SyncResult = { at: string; ok: boolean; entitlements: number; changed: number; error?: string }; let lastSync: SyncResult | null = null; +let consecutiveFailures = 0; +/** Failed syncs in a row (2 min apart) before an incident: about 6 minutes. */ +const FAILURES_BEFORE_INCIDENT = 3; export const lastDiscordSync = () => lastSync; export async function configuredSkus() { @@ -33,9 +38,20 @@ export async function syncDiscordPlans(log: Logger): Promise { } catch (error) { log.warn({ err: error }, 'Unable to read Discord entitlements'); lastSync = { at: new Date().toISOString(), ok: false, entitlements: 0, changed: 0, error: 'ENTITLEMENTS_UNAVAILABLE' }; + consecutiveFailures += 1; + if (consecutiveFailures >= FAILURES_BEFORE_INCIDENT) { + await openIncident('discord.sync', { + category: 'billing', + severity: 'warning', + title: 'Sincronizzazione abbonamenti Discord ferma', + description: 'Gli abbonamenti attivi non si possono leggere da Discord: nuovi acquisti, rinnovi e disdette non arrivano ai piani dei server.' + }).catch(() => null); + } return lastSync; } + consecutiveFailures = 0; + await resolveIncident('discord.sync', { title: 'Sincronizzazione abbonamenti ripristinata', description: (duration) => `Gli abbonamenti Discord tornano ad arrivare ai piani dopo ${duration}.` }).catch(() => null); const now = new Date(); const discordPlans = plansFromEntitlements(entitlements, skus, now); const rows = await prisma.guildSettings.findMany({ @@ -68,6 +84,12 @@ export async function syncDiscordPlans(log: Logger): Promise { await systemAudit('plan.discord_subscription', 'GUILD', row.guildId, { previousTier: row.planTier, tier: plan.tier, billing: plan.billing, entitlementId: plan.entitlementId, applied }); + await recordAlert({ + category: 'billing', severity: 'info', level: 'ok', + title: row.planSource === 'DISCORD' && row.planTier !== plan.tier ? 'Abbonamento Discord cambiato' : 'Nuovo abbonamento Discord', + description: `Il server ora ha il piano ${PLANS[plan.tier].name}.`, + fields: [{ name: 'Server', value: `\`${row.guildId}\`` }] + }).catch(() => null); } } else { await prisma.guildSettings.update({ @@ -76,6 +98,12 @@ export async function syncDiscordPlans(log: Logger): Promise { }); const applied = await applyPlanLimits(row.guildId, 'FREE', log); await systemAudit('plan.discord_ended', 'GUILD', row.guildId, { previousTier: row.planTier, entitlementId: row.planEntitlementId, applied }); + await recordAlert({ + category: 'billing', severity: 'info', + title: 'Abbonamento Discord terminato', + description: 'Il server è tornato al piano Free.', + fields: [{ name: 'Server', value: `\`${row.guildId}\`` }] + }).catch(() => null); } changed += 1; } catch (error) { diff --git a/apps/api/src/discord.ts b/apps/api/src/discord.ts index be06dc9..694b3ef 100644 --- a/apps/api/src/discord.ts +++ b/apps/api/src/discord.ts @@ -98,6 +98,14 @@ export async function getEntitlements() { return api('/entitlements', 'GET', undefined, 15_000); } +/** Posts a routed alert through the bot to a channel of a server it is in. */ +export async function deliverAlert(alert: { + guildId: string; channelId: string; title: string; description: string | null; + fields: Array<{ name: string; value: string }>; color: number; footer: string; mentionUserId: string | null; +}) { + return api<{ ok: true }>('/alerts/deliver', 'POST', alert, 10_000); +} + export async function sendStatusTest() { return api<{ ok: true } | { ok: false; error: string }>('/status/test', 'POST', {}, 10_000); } diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index b765388..e85e6b3 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -8,6 +8,7 @@ import { prisma } from '@sentinel/db'; import { EVENT_CATALOG, createLogBuffer, tierAllows } from '@sentinel/shared'; import pino from 'pino'; import { registerMonitoringRoutes, startMonitoring } from './monitoring.js'; +import { registerAlertRoutes, startAlerting } from './alerting.js'; import { config } from './config.js'; import { createSession, destroySession, isSuperAdminUserId, randomToken, requireGuild, requireSession, requireSuperAdmin, resolveGuildAccess, type OAuthGuild } from './auth.js'; import { getBotGuilds, getGuildResources, leaveBotGuild } from './discord.js'; @@ -792,7 +793,9 @@ app.get('/api/guilds/:guildId/export', async (request, reply) => { await registerPlanRoutes(app); await registerMonitoringRoutes(app, apiLogs); +await registerAlertRoutes(app); startMonitoring(app.log); +startAlerting(app.log); const shutdown = async (signal: string) => { app.log.info({ signal }, 'Shutting down'); diff --git a/apps/api/src/monitoring.ts b/apps/api/src/monitoring.ts index 33b7702..b6cdff5 100644 --- a/apps/api/src/monitoring.ts +++ b/apps/api/src/monitoring.ts @@ -4,19 +4,20 @@ import { prisma } from '@sentinel/db'; import { LOG_LEVEL_RANK, type LogBuffer, type LogLevelName } from '@sentinel/shared'; import { requireSuperAdmin } from './auth.js'; import { botHealth, getBotLogs } from './discord.js'; -import { decryptText, encryptText, protectJson, unprotectJson } from './security.js'; +import { encryptText, unprotectJson } from './security.js'; import { superAdminAudit } from './super-audit.js'; +import { alertWebhookUrl, openIncident, postAlertWebhook, resolveIncident } from './alerting.js'; /** - * Super console monitoring: the bot watchdog with "down" and "restored" - * alarms, the instance event feed and the latest logs of the API and the bot. + * Super console monitoring: the bot watchdog, the instance event feed and the + * latest logs of the API and the bot. * - * Alarms go to a Discord webhook, not through the bot: they must reach you - * precisely when the bot is not running. + * "Bot down" cannot be posted by the bot: it is recorded as an incident, sent + * to the optional webhook and caught by the external monitor on + * /health/bot. "Bot restored" is routed like any other alert. */ type Logger = { info: (obj: object, msg: string) => void; warn: (obj: object, msg: string) => void }; -type Level = 'info' | 'ok' | 'warn' | 'error'; const CHECK_INTERVAL_MS = 30_000; /** @@ -26,47 +27,14 @@ const CHECK_INTERVAL_MS = 30_000; const FAILURES_BEFORE_DOWN = 4; const EVENT_RETENTION_DAYS = 90; const WEBHOOK = /^https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/\d{17,20}\/[\w-]{20,}$/; -const COLORS: Record = { info: 0xa78bfa, ok: 0x3ccf8e, warn: 0xf5a524, error: 0xf2555a }; type BotState = { status: 'unknown' | 'up' | 'down'; since: string; lastCheck: string | null; failures: number; guilds: number | null }; const state: BotState = { status: 'unknown', since: new Date().toISOString(), lastCheck: null, failures: 0, guilds: null }; export const botState = () => ({ ...state }); -const formatDuration = (ms: number) => { - const minutes = Math.round(ms / 60_000); - if (minutes < 60) return `${minutes} min`; - const hours = Math.floor(minutes / 60); - return `${hours} h ${minutes % 60} min`; -}; - -export async function recordEvent(level: Level, title: string, description: string | null = null) { - await prisma.instanceEvent.create({ data: { source: 'api', level, title, details: protectJson({ description, fields: [] }) } }); -} - -async function webhookUrl() { - const row = await prisma.instanceConfig.findUnique({ where: { id: 1 }, select: { alertWebhookUrl: true } }); - return row?.alertWebhookUrl ? decryptText(row.alertWebhookUrl) : null; -} - -async function postWebhook(url: string, level: Level, title: string, description: string) { - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - username: 'Sentinel monitor', - allowed_mentions: { parse: [] }, - embeds: [{ title, description, color: COLORS[level], timestamp: new Date().toISOString(), footer: { text: 'Sentinel · monitoraggio' } }] - }), - signal: AbortSignal.timeout(8_000) - }); - if (!response.ok) throw new Error(`Webhook responded ${response.status}`); -} - -async function alarm(level: Level, title: string, description: string, log: Logger) { - await recordEvent(level, title, description).catch((error) => log.warn({ err: error }, 'Unable to record instance event')); - const url = await webhookUrl().catch(() => null); - if (!url) return; - await postWebhook(url, level, title, description).catch((error) => log.warn({ err: error }, 'Alarm webhook failed')); +async function webhook(level: 'ok' | 'error', title: string, description: string, log: Logger) { + const url = await alertWebhookUrl().catch(() => null); + if (url) await postAlertWebhook(url, level, title, description).catch((error) => log.warn({ err: error }, 'Alarm webhook failed')); } async function checkBot(log: Logger) { @@ -76,14 +44,16 @@ async function checkBot(log: Logger) { if (health?.ok) { state.guilds = health.guilds; state.failures = 0; - if (state.status === 'down') { - const downtime = formatDuration(now.getTime() - new Date(state.since).getTime()); - await alarm('ok', 'Bot ripristinato', `Sentinel è di nuovo connesso a Discord dopo ${downtime} di interruzione (${health.guilds} server).`, log); - } if (state.status !== 'up') { state.status = 'up'; state.since = now.toISOString(); } + // Also closes an incident left open by a previous API process. + const downtime = await resolveIncident('bot.down', { + title: 'Bot ripristinato', + description: (duration) => `Sentinel è di nuovo connesso a Discord dopo ${duration} di interruzione (${health.guilds} server).` + }).catch(() => null); + if (downtime) await webhook('ok', 'Bot ripristinato', `Sentinel è di nuovo connesso a Discord dopo ${downtime} di interruzione.`, log); return; } state.failures += 1; @@ -91,9 +61,12 @@ async function checkBot(log: Logger) { const wasUp = state.status === 'up'; state.status = 'down'; state.since = now.toISOString(); - await alarm('error', 'Bot non raggiungibile', health + const description = health ? 'Il processo del bot risponde ma non è connesso a Discord: i log non vengono registrati.' - : `Il bot non risponde da circa ${Math.round(FAILURES_BEFORE_DOWN * CHECK_INTERVAL_MS / 60_000)} minuti${wasUp ? '' : ' (dall’avvio dell’API)'}: i log non vengono registrati.`, log); + : `Il bot non risponde da circa ${Math.round(FAILURES_BEFORE_DOWN * CHECK_INTERVAL_MS / 60_000)} minuti${wasUp ? '' : ' (dall’avvio dell’API)'}: i log non vengono registrati.`; + // logOnly: the bot cannot post it; the webhook and the external monitor do. + const opened = await openIncident('bot.down', { category: 'availability', severity: 'critical', title: 'Bot non raggiungibile', description, logOnly: true }).catch(() => false); + if (opened) await webhook('error', 'Bot non raggiungibile', description, log); } } @@ -117,7 +90,7 @@ export async function registerMonitoringRoutes(app: FastifyInstance, apiLogs: Lo if (!session) return; const [rows, url] = await Promise.all([ prisma.instanceEvent.findMany({ orderBy: { createdAt: 'desc' }, take: 200 }), - webhookUrl().catch(() => null) + alertWebhookUrl().catch(() => null) ]); const events = rows.map((row) => { let details: { description?: string | null; fields?: Array<{ name: string; value: string }> } = {}; @@ -126,7 +99,7 @@ export async function registerMonitoringRoutes(app: FastifyInstance, apiLogs: Lo } catch { details = { description: null, fields: [] }; } - return { id: row.id, source: row.source, level: row.level, title: row.title, description: details.description ?? null, fields: details.fields ?? [], createdAt: row.createdAt }; + return { id: row.id, source: row.source, level: row.level, category: row.category, severity: row.severity, deliveryState: row.deliveryState, title: row.title, description: details.description ?? null, fields: details.fields ?? [], createdAt: row.createdAt }; }); return { bot: botState(), events, webhook: url ? { configured: true, hint: `…${url.slice(-6)}` } : { configured: false, hint: null } }; }); @@ -164,10 +137,10 @@ export async function registerMonitoringRoutes(app: FastifyInstance, apiLogs: Lo app.post('/api/super/alert-webhook/test', async (request, reply) => { const session = await requireSuperAdmin(request, reply); if (!session) return; - const url = await webhookUrl().catch(() => null); + const url = await alertWebhookUrl().catch(() => null); if (!url) return reply.code(400).send({ error: 'WEBHOOK_NOT_SET' }); try { - await postWebhook(url, 'info', 'Allarmi attivi', 'Messaggio di prova dalla super console: qui arriveranno gli avvisi quando il bot non è raggiungibile e quando torna attivo.'); + await postAlertWebhook(url, 'info', 'Allarmi attivi', 'Messaggio di prova dalla super console: qui arriveranno l’avviso e i promemoria quando il bot non è raggiungibile, e la notizia del ripristino.'); } catch { return reply.code(502).send({ error: 'WEBHOOK_FAILED' }); } diff --git a/apps/api/test/alert-rules.test.ts b/apps/api/test/alert-rules.test.ts new file mode 100644 index 0000000..9ad15bc --- /dev/null +++ b/apps/api/test/alert-rules.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + DEFAULT_ALERT_SETTINGS, + digestDue, + formatDuration, + groupForDelivery, + inQuietHours, + parseAlertSettings, + reminderDue, + routeTarget +} from '../src/alert-rules.ts'; + +test('settings merge over the defaults and reject invalid values', () => { + assert.deepEqual(parseAlertSettings(null), DEFAULT_ALERT_SETTINGS); + const settings = parseAlertSettings({ + routes: { + errors: { mode: 'channel', guildId: '100000000000000001', channelId: '200000000000000001', mention: true }, + usage: { mode: 'log' }, + billing: { mode: 'channel', guildId: 'x' }, + nope: { mode: 'status' } + }, + quietHours: { enabled: true, start: '25:00', end: '07:30' }, + digest: { time: '18:00', mode: 'channel', guildId: '100000000000000001', channelId: '200000000000000002' }, + timezone: 'Mars/Olympus' + }); + assert.deepEqual(settings.routes.errors, { mode: 'channel', guildId: '100000000000000001', channelId: '200000000000000001', mention: true }); + assert.equal(settings.routes.usage.mode, 'log'); + assert.deepEqual(settings.routes.billing, { mode: 'channel', guildId: null, channelId: null, mention: false }); + assert.equal(settings.routes.availability.mention, true); + assert.deepEqual(settings.quietHours, { enabled: true, start: '23:00', end: '07:30' }); + assert.equal(settings.digest.channelId, '200000000000000002'); + assert.equal(settings.timezone, 'Europe/Rome'); +}); + +test('routes resolve to a channel or to nothing', () => { + const status = { guildId: '1', channelId: '2' }; + assert.deepEqual(routeTarget({ mode: 'status', guildId: null, channelId: null }, status), status); + assert.equal(routeTarget({ mode: 'status', guildId: null, channelId: null }, null), null); + assert.deepEqual(routeTarget({ mode: 'channel', guildId: '3', channelId: '4' }, status), { guildId: '3', channelId: '4' }); + assert.equal(routeTarget({ mode: 'digest', guildId: null, channelId: null }, status), null); +}); + +test('quiet hours cross midnight in the configured time zone', () => { + const settings = { ...DEFAULT_ALERT_SETTINGS, quietHours: { enabled: true, start: '23:00', end: '08:00' } }; + // Rome is UTC+2 in October. + assert.equal(inQuietHours(settings, new Date('2026-10-21T22:30:00Z')), true); + assert.equal(inQuietHours(settings, new Date('2026-10-21T05:59:00Z')), true); + assert.equal(inQuietHours(settings, new Date('2026-10-21T06:00:00Z')), false); + assert.equal(inQuietHours(settings, new Date('2026-10-21T12:00:00Z')), false); + assert.equal(inQuietHours({ ...settings, quietHours: { ...settings.quietHours, enabled: false } }, new Date('2026-10-21T22:30:00Z')), false); +}); + +test('the digest is due once per local day after its time', () => { + const settings = DEFAULT_ALERT_SETTINGS; // 09:00 Europe/Rome + assert.equal(digestDue(settings, new Date('2026-10-21T06:59:00Z'), null), false); + assert.equal(digestDue(settings, new Date('2026-10-21T07:00:00Z'), null), true); + assert.equal(digestDue(settings, new Date('2026-10-21T15:00:00Z'), new Date('2026-10-21T07:00:30Z')), false); + assert.equal(digestDue(settings, new Date('2026-10-22T07:05:00Z'), new Date('2026-10-21T07:00:30Z')), true); +}); + +test('repeated alerts are grouped and held during the window', () => { + const now = 1_000_000_000; + const lastSent = new Map([['a', now - 60_000], ['b', now - 11 * 60_000]]); + const groups = groupForDelivery([{ key: 'a', id: 1 }, { key: 'b', id: 2 }, { key: 'b', id: 3 }, { key: 'c', id: 4 }], lastSent, now); + assert.deepEqual(groups.map((group) => [group.key, group.members.length]), [['b', 2], ['c', 1]]); +}); + +test('reminders after 15 minutes, 1 hour, then every 6 hours', () => { + const openedAt = new Date('2026-10-21T10:00:00Z'); + const base = { severity: 'warning', openedAt, resolvedAt: null, acknowledgedAt: null, lastRemindedAt: null, reminders: 0 }; + assert.equal(reminderDue(base, new Date('2026-10-21T10:14:00Z')), false); + assert.equal(reminderDue(base, new Date('2026-10-21T10:15:00Z')), true); + assert.equal(reminderDue({ ...base, reminders: 1, lastRemindedAt: new Date('2026-10-21T10:15:00Z') }, new Date('2026-10-21T10:59:00Z')), false); + assert.equal(reminderDue({ ...base, reminders: 1 }, new Date('2026-10-21T11:00:00Z')), true); + assert.equal(reminderDue({ ...base, reminders: 2, lastRemindedAt: new Date('2026-10-21T11:00:00Z') }, new Date('2026-10-21T16:59:00Z')), false); + assert.equal(reminderDue({ ...base, reminders: 2, lastRemindedAt: new Date('2026-10-21T11:00:00Z') }, new Date('2026-10-21T17:00:00Z')), true); + assert.equal(reminderDue({ ...base, acknowledgedAt: openedAt }, new Date('2026-10-22T10:00:00Z')), false); + assert.equal(reminderDue({ ...base, severity: 'info' }, new Date('2026-10-22T10:00:00Z')), false); + assert.equal(formatDuration(90 * 60_000), '1 h 30 min'); +}); diff --git a/apps/bot/src/dispatcher.ts b/apps/bot/src/dispatcher.ts index 9026fb7..79ca422 100644 --- a/apps/bot/src/dispatcher.ts +++ b/apps/bot/src/dispatcher.ts @@ -173,10 +173,13 @@ export function startDispatcher(client: Client) { * servers being logged. Like log delivery, the target must belong to the * configured guild. */ -export async function sendStatusMessage(client: Client, guildId: string, channelId: string, embed: EmbedBuilder) { +export async function sendStatusMessage(client: Client, guildId: string, channelId: string, embed: EmbedBuilder, mentionUserId: string | null = null) { const guild = client.guilds.cache.get(guildId); if (!guild) throw new Error('STATUS_GUILD_UNAVAILABLE'); const channel = await guild.channels.fetch(channelId).catch(() => null); if (!channel || channel.guildId !== guild.id || !channel.isSendable()) throw new Error('STATUS_CHANNEL_UNAVAILABLE'); - await channel.send({ embeds: [embed], allowedMentions: { parse: [] } }); + // Only critical alerts mention someone: the instance owner, by ID. + await channel.send(mentionUserId + ? { content: `<@${mentionUserId}>`, embeds: [embed], allowedMentions: { parse: [], users: [mentionUserId] } } + : { embeds: [embed], allowedMentions: { parse: [] } }); } diff --git a/apps/bot/src/handlers.ts b/apps/bot/src/handlers.ts index a34e397..630671e 100644 --- a/apps/bot/src/handlers.ts +++ b/apps/bot/src/handlers.ts @@ -96,6 +96,8 @@ export function registerHandlers(client: Client, status: StatusNotifier) { }[reason]; status.notify({ level: 'warn', + category: 'security', + severity: 'warning', title: 'Installazione rifiutata', description: `Sentinel è uscito da **${guild.name}**: ${why}.`, fields: [ @@ -124,6 +126,8 @@ export function registerHandlers(client: Client, status: StatusNotifier) { await ensureGuild(guild); status.notify({ level: 'ok', + category: 'installations', + severity: 'info', title: 'Sentinel aggiunto a un server', fields: [ { name: 'Server', value: `${guild.name}\n\`${guild.id}\`` }, @@ -744,6 +748,8 @@ export function registerHandlers(client: Client, status: StatusNotifier) { if (!intentionalLeaves.delete(guild.id)) { status.notify({ level: 'info', + category: 'installations', + severity: 'info', title: 'Sentinel rimosso da un server', fields: [{ name: 'Server', value: `${guild.name ?? 'sconosciuto'}\n\`${guild.id}\`` }] }); diff --git a/apps/bot/src/index.ts b/apps/bot/src/index.ts index 0e5a81a..9f5519c 100644 --- a/apps/bot/src/index.ts +++ b/apps/bot/src/index.ts @@ -43,6 +43,15 @@ const client = new Client({ installDiscordCapabilityPolicy(client, ({ method, route }) => { logger.error({ method, route }, 'Blocked Discord mutation by capability policy'); + // Never expected: some code tried a mutation outside the read-only policy. + status.notify({ + level: 'error', + category: 'security', + severity: 'critical', + title: 'Operazione bloccata dalla politica di sicurezza', + description: `Il bot ha tentato una chiamata non consentita e la politica l’ha bloccata: \`${method} ${route}\`.`, + dedupeKey: `policy:${method}:${route}` + }); }); const status = createStatusNotifier(client); @@ -51,6 +60,8 @@ onQuotaExceeded((guildId, limit, tier) => { const guild = client.guilds.cache.get(guildId); status.notify({ level: 'warn', + category: 'usage', + severity: 'info', title: 'Quota giornaliera raggiunta', description: `**${guild?.name ?? guildId}** ha superato ${limit.toLocaleString('it-IT')} eventi oggi (piano ${tier}): gli eventi successivi non vengono registrati fino a mezzanotte UTC.`, fields: [{ name: 'Server', value: `\`${guildId}\`` }], @@ -82,6 +93,8 @@ client.once(Events.ClientReady, async (ready) => { } status.notify({ level: 'ok', + category: 'availability', + severity: 'info', title: 'Sentinel online', description: `Connesso a Discord come **${ready.user.tag}**.`, fields: [ @@ -105,6 +118,8 @@ async function runMaintenance() { logger.info({ count: purged.length }, 'Deleted data of removed servers'); status.notify({ level: 'info', + category: 'installations', + severity: 'info', title: 'Dati di server rimossi cancellati', description: `Cancellati i dati di ${purged.length} server da cui Sentinel è stato rimosso da più di ${REMOVED_GUILD_GRACE_DAYS} giorni.`, fields: [{ name: 'Server', value: purged.slice(0, 20).map((id) => `\`${id}\``).join('\n') }] @@ -119,16 +134,16 @@ async function runMaintenance() { // console, not to the servers being logged. client.on(Events.Warn, (warning) => { logger.warn({ warning: redactText(warning) }, 'Discord client warning'); - status.notify({ level: 'warn', title: 'Warning del client Discord', description: redactText(warning) }); + status.notify({ level: 'warn', category: 'errors', severity: 'warning', title: 'Warning del client Discord', description: redactText(warning) }); }); client.on(Events.Error, (error) => { logger.error({ error: errorText(error) }, 'Discord client error'); - status.notify({ level: 'error', title: 'Errore del client Discord', description: errorText(error) }); + status.notify({ level: 'error', category: 'errors', severity: 'warning', title: 'Errore del client Discord', description: errorText(error) }); }); client.on(Events.ShardResume, (shardId, replayed) => { - status.notify({ level: 'info', title: 'Connessione a Discord ripristinata', description: `Shard ${shardId}: ${replayed} eventi recuperati.`, dedupeKey: 'shard-resume' }); + status.notify({ level: 'info', category: 'availability', severity: 'info', title: 'Connessione a Discord ripristinata', description: `Shard ${shardId}: ${replayed} eventi recuperati.`, dedupeKey: 'shard-resume' }); }); setInterval(() => void runMaintenance(), 6 * 60 * 60 * 1000).unref(); @@ -156,7 +171,7 @@ const shutdown = async (signal: string) => { process.on('unhandledRejection', (reason) => { logger.error({ reason: errorText(reason) }, 'Unhandled promise rejection'); - status.notify({ level: 'error', title: 'Errore non gestito nel bot', description: errorText(reason) }); + status.notify({ level: 'error', category: 'errors', severity: 'warning', title: 'Errore non gestito nel bot', description: errorText(reason) }); }); process.on('uncaughtException', (error) => { diff --git a/apps/bot/src/internal-api.ts b/apps/bot/src/internal-api.ts index 473e762..4f24a10 100644 --- a/apps/bot/src/internal-api.ts +++ b/apps/bot/src/internal-api.ts @@ -1,10 +1,13 @@ import crypto from 'node:crypto'; import http from 'node:http'; -import { DiscordAPIError, Routes, type Client } from 'discord.js'; +import { DiscordAPIError, EmbedBuilder, Routes, type Client } from 'discord.js'; import { editOwnGuildProfile, leaveGuild } from './discord-actions.js'; import type { StatusNotifier } from './status.js'; import { LOG_LEVEL_RANK, type LogLevelName } from '@sentinel/shared'; import { logBuffer } from './logger.js'; +import { sendStatusMessage } from './dispatcher.js'; +import { truncate } from './embed-format.js'; +import { redactText } from './security.js'; const SNOWFLAKE = /^\d{17,20}$/; @@ -186,11 +189,42 @@ export function startInternalApi(client: Client, secret: string, port = 3002, st return; } + // Alerts routed by the API (super console → Monitoring) to a channel of a + // server the bot is in. The channel check matches log delivery. + if (req.method === 'POST' && url.pathname === '/alerts/deliver') { + const body = await readJson(req); + const guildId = String(body.guildId ?? ''); + const channelId = String(body.channelId ?? ''); + const mention = body.mentionUserId == null ? null : String(body.mentionUserId); + if (!SNOWFLAKE.test(guildId) || !SNOWFLAKE.test(channelId) || (mention !== null && !SNOWFLAKE.test(mention))) throw new RequestError(400, 'INVALID_TARGET'); + const embed = new EmbedBuilder() + .setTitle(truncate(String(body.title ?? 'Sentinel'), 256)) + .setColor(Number(body.color) || 0xa78bfa) + .setFooter({ text: truncate(String(body.footer ?? 'Sentinel · allarmi'), 200) }) + .setTimestamp(new Date()); + if (body.description) embed.setDescription(truncate(redactText(String(body.description)), 4000)); + const fields = Array.isArray(body.fields) ? body.fields as Array<{ name?: unknown; value?: unknown }> : []; + const safeFields = fields.filter((field) => field?.value).slice(0, 20) + .map((field) => ({ name: truncate(String(field.name ?? '—'), 256), value: truncate(redactText(String(field.value)), 1024), inline: true })); + if (safeFields.length) embed.addFields(safeFields); + try { + await sendStatusMessage(client, guildId, channelId, embed, mention); + } catch (error) { + res.statusCode = 502; + res.end(JSON.stringify({ error: error instanceof Error ? error.message.slice(0, 200) : 'SEND_FAILED' })); + return; + } + res.end('{"ok":true}'); + return; + } + if (req.method === 'POST' && url.pathname === '/status/test') { if (!status) throw new Error('STATUS_UNAVAILABLE'); status.invalidate(); const result = await status.deliver({ level: 'info', + category: 'availability', + severity: 'info', title: 'Notifiche di stato attive', description: 'Messaggio di prova inviato dalla super console. Qui arriveranno avvio del bot, errori e server aggiunti o rimossi.' }, { force: true }); diff --git a/apps/bot/src/status.ts b/apps/bot/src/status.ts index a3b539e..c88c8f1 100644 --- a/apps/bot/src/status.ts +++ b/apps/bot/src/status.ts @@ -3,12 +3,17 @@ import { prisma } from '@sentinel/db'; import { sendStatusMessage } from './dispatcher.js'; import { truncate } from './embed-format.js'; import { logger } from './logger.js'; +import type { AlertCategory, AlertSeverity } from '@sentinel/shared'; import { protectJson, redactText } from './security.js'; export type StatusLevel = 'info' | 'ok' | 'warn' | 'error'; export type StatusNotice = { level: StatusLevel; + /** Decides where the API routes it (super console → Monitoring). */ + category: AlertCategory; + /** critical: mention and reminders, ignores quiet hours; warning; info. */ + severity: AlertSeverity; title: string; description?: string; fields?: Array<{ name: string; value: string; inline?: boolean }>; @@ -19,9 +24,12 @@ export type StatusNotice = { export type StatusResult = { ok: true } | { ok: false; error: string }; export type StatusNotifier = { - /** Sends and reports the outcome (used by the super console test button). */ + /** Sends straight to the status channel (the super console test button). */ deliver: (notice: StatusNotice, options?: { force?: boolean }) => Promise; - /** Fire-and-forget: status notices must never break the code path that raises them. */ + /** + * Fire-and-forget: records the notice; the API routes it to its channel, + * groups repeats and applies quiet hours. Must never break the caller. + */ notify: (notice: StatusNotice) => void; /** Drops the cached target after the super console changes it. */ invalidate: () => void; @@ -30,10 +38,10 @@ export type StatusNotifier = { const COLORS: Record = { info: 0xa78bfa, ok: 0x3ccf8e, warn: 0xf5a524, error: 0xf2555a }; const CONFIG_CACHE_MS = 60_000; const DEDUPE_MS = 10 * 60_000; -// A crash loop or a flood of Gateway warnings must not turn into a flood of -// Discord messages (and a rate limit that also delays the real logs). +// A crash loop or a flood of Gateway warnings must not flood the database; +// repeats that get through are merged by the API before reaching Discord. const BURST_WINDOW_MS = 10 * 60_000; -const BURST_MAX = 30; +const BURST_MAX = 120; type Target = { guildId: string; channelId: string } | null; @@ -53,10 +61,11 @@ export function createStatusNotifier(client: Client): StatusNotifier { const throttled = (notice: StatusNotice) => { const now = Date.now(); - const key = notice.dedupeKey ?? `${notice.level}:${notice.title}:${notice.description ?? ''}`; - const last = recent.get(key); - if (last && now - last < DEDUPE_MS) return true; - recent.set(key, now); + if (notice.dedupeKey) { + const last = recent.get(notice.dedupeKey); + if (last && now - last < DEDUPE_MS) return true; + recent.set(notice.dedupeKey, now); + } if (recent.size > 500) { for (const [entry, at] of recent) if (now - at >= DEDUPE_MS) recent.delete(entry); } @@ -77,13 +86,12 @@ export function createStatusNotifier(client: Client): StatusNotifier { const fields = (notice.fields ?? []).filter((field) => field.value).slice(0, 20) .map((field) => ({ name: field.name.slice(0, 256), value: redactText(field.value).slice(0, 1024) })); await prisma.instanceEvent.createMany({ - data: [{ source: 'bot', level: notice.level, title: notice.title.slice(0, 256), details: protectJson({ description, fields }) }] + data: [{ source: 'bot', level: notice.level, category: notice.category, severity: notice.severity, title: notice.title.slice(0, 256), details: protectJson({ description, fields }) }] }).catch((error: unknown) => logger.warn({ error: redactText(String(error)) }, 'Unable to record instance event')); }; const deliver: StatusNotifier['deliver'] = async (notice, options = {}) => { if (!options.force && throttled(notice)) return { ok: false, error: 'THROTTLED' }; - if (!options.force) await record(notice); let config: Target; try { config = await target(); @@ -116,7 +124,7 @@ export function createStatusNotifier(client: Client): StatusNotifier { return { deliver, - notify: (notice) => { void deliver(notice).catch(() => null); }, + notify: (notice) => { if (!throttled(notice)) void record(notice).catch(() => null); }, invalidate: () => { cached = null; } }; } diff --git a/apps/web/app/_components/SuperConsole.tsx b/apps/web/app/_components/SuperConsole.tsx index d7e03b3..ff2ed3b 100644 --- a/apps/web/app/_components/SuperConsole.tsx +++ b/apps/web/app/_components/SuperConsole.tsx @@ -88,6 +88,37 @@ const copy = { url: 'Link: server Discord, sito o modulo (https)', save: 'Salva recapiti' }, + alerts: { + routingTitle: 'Instradamento degli allarmi', + routingText: 'Scegli dove va ogni categoria. Gli avvisi uguali entro 10 minuti diventano un solo messaggio; i critici ti menzionano e ignorano gli orari di silenzio. “Bot giù” lo segnala il monitor esterno (e il webhook, se impostato).', + categories: { availability: 'Disponibilità', security: 'Sicurezza', errors: 'Errori', delivery: 'Consegna dei log', billing: 'Abbonamenti', installations: 'Installazioni', usage: 'Utilizzo' }, + modes: { status: 'Canale di stato', channel: 'Altro canale', digest: 'Riepilogo giornaliero', log: 'Solo registro' }, + server: 'Server', + channel: 'Canale', + choose: 'Scegli…', + mention: 'Menzionami per i critici', + quiet: 'Orari di silenzio', + quietText: 'In questa fascia arrivano solo gli avvisi critici; gli altri partono tutti insieme alla fine.', + from: 'Dalle', + to: 'alle', + timezone: 'Fuso orario', + digest: 'Riepilogo giornaliero', + digestText: 'Un messaggio al giorno con il conteggio degli avvisi informativi.', + digestAt: 'Ora di invio', + digestWhere: 'Destinazione', + save: 'Salva instradamento', + incidents: 'Incidenti', + incidentsText: 'Problemi che durano: restano aperti finché non si risolvono e mandano promemoria dopo 15 minuti, dopo un’ora e poi ogni 6 ore, finché non li segni come presi in carico.', + open: 'Aperti', + none: 'Nessun incidente aperto.', + recent: 'Risolti di recente', + ack: 'Preso in carico', + acked: 'preso in carico da', + openedAt: 'aperto', + resolvedAt: 'risolto', + reminders: 'promemoria', + states: { pending: 'in attesa', sent: 'inviato', digest: 'nel riepilogo', digested: 'riepilogato', log: 'solo registro', failed: 'non consegnato' } + }, monitor: { title: 'Stato e allarmi', text: 'L’API controlla il bot ogni 30 secondi. Se non risponde per circa due minuti invia un allarme al webhook qui sotto, e un altro quando torna attivo. Gli avvisi passano da un webhook perché devono arrivare proprio quando il bot è fermo.', @@ -138,7 +169,7 @@ const copy = { }, status: { title: 'Canale notifiche di stato', - text: 'Il bot invia qui avvio, errori, warning, quote raggiunte e server aggiunti, rimossi o rifiutati. Non sono più visibili nei singoli server.', + text: 'Destinazione predefinita degli allarmi: in Monitoraggio scegli quali categorie arrivano qui, in un altro canale o nel riepilogo giornaliero. Non sono mai visibili nei server registrati.', guild: 'Server', channel: 'Canale', choose: 'Scegli…', @@ -176,6 +207,7 @@ const copy = { errors: { INVALID_SKU: 'ID SKU non valido (17-20 cifre).', INVALID_WEBHOOK: 'URL del webhook non valido: copialo da Integrazioni → Webhook.', + CHANNEL_REQUIRED: 'Scegli server e canale per ogni categoria impostata su “Altro canale”.', WEBHOOK_NOT_SET: 'Nessun webhook configurato.', WEBHOOK_FAILED: 'Discord ha rifiutato il messaggio: controlla che il webhook esista ancora.', DUPLICATE_SKU: 'Lo stesso SKU non può valere per due piani.', @@ -228,6 +260,37 @@ const copy = { url: 'Link: Discord server, website or form (https)', save: 'Save contacts' }, + alerts: { + routingTitle: 'Alert routing', + routingText: 'Choose where each category goes. Identical alerts within 10 minutes become a single message; critical ones mention you and ignore quiet hours. “Bot down” is reported by the external monitor (and by the webhook, if set).', + categories: { availability: 'Availability', security: 'Security', errors: 'Errors', delivery: 'Log delivery', billing: 'Subscriptions', installations: 'Installations', usage: 'Usage' }, + modes: { status: 'Status channel', channel: 'Another channel', digest: 'Daily digest', log: 'Log only' }, + server: 'Server', + channel: 'Channel', + choose: 'Choose…', + mention: 'Mention me for critical alerts', + quiet: 'Quiet hours', + quietText: 'Only critical alerts arrive in this window; the others leave together when it ends.', + from: 'From', + to: 'to', + timezone: 'Time zone', + digest: 'Daily digest', + digestText: 'One message a day with the count of informational alerts.', + digestAt: 'Send at', + digestWhere: 'Destination', + save: 'Save routing', + incidents: 'Incidents', + incidentsText: 'Problems that last: they stay open until resolved and send reminders after 15 minutes, after an hour and then every 6 hours, until you acknowledge them.', + open: 'Open', + none: 'No open incidents.', + recent: 'Recently resolved', + ack: 'Acknowledge', + acked: 'acknowledged by', + openedAt: 'opened', + resolvedAt: 'resolved', + reminders: 'reminders', + states: { pending: 'pending', sent: 'sent', digest: 'in digest', digested: 'digested', log: 'log only', failed: 'not delivered' } + }, monitor: { title: 'Status and alarms', text: 'The API checks the bot every 30 seconds. If it does not answer for about two minutes, an alarm is posted to the webhook below, and another one when it is back. Alarms use a webhook because they must arrive precisely when the bot is not running.', @@ -278,7 +341,7 @@ const copy = { }, status: { title: 'Status notification channel', - text: 'The bot posts startup, errors, warnings, reached quotas and servers added, removed or rejected here. They are no longer visible in individual servers.', + text: 'Default destination for alerts: in Monitoring choose which categories arrive here, in another channel or in the daily digest. They are never visible in the logged servers.', guild: 'Server', channel: 'Channel', choose: 'Choose…', @@ -316,6 +379,7 @@ const copy = { errors: { INVALID_SKU: 'Invalid SKU ID (17-20 digits).', INVALID_WEBHOOK: 'Invalid webhook URL: copy it from Integrations → Webhooks.', + CHANNEL_REQUIRED: 'Choose a server and a channel for every category set to “Another channel”.', WEBHOOK_NOT_SET: 'No webhook configured.', WEBHOOK_FAILED: 'Discord rejected the message: check that the webhook still exists.', DUPLICATE_SKU: 'The same SKU cannot be used for two plans.', @@ -434,7 +498,7 @@ export default function SuperConsole({ locale }: { locale: Locale }) { {tab === 'servers' && } - {tab === 'monitor' && } + {tab === 'monitor' && } {tab === 'logs' && } {tab === 'contacts' && } {tab === 'discord' && } @@ -574,14 +638,14 @@ function PlanEditor({ guild, c, locale, action, onDone }: { guild: Guild; c: Cop /* ------------------------------------------------------------------ monitor */ -type MonitorEvent = { id: string; source: string; level: string; title: string; description: string | null; fields: Array<{ name: string; value: string }>; createdAt: string }; +type MonitorEvent = { id: string; source: string; level: string; category: string; severity: string; deliveryState: string; title: string; description: string | null; fields: Array<{ name: string; value: string }>; createdAt: string }; type Monitor = { bot: { status: 'unknown' | 'up' | 'down'; since: string; lastCheck: string | null; guilds: number | null }; events: MonitorEvent[]; webhook: { configured: boolean; hint: string | null }; }; -function MonitorTab({ c, action, fmt, setStatus }: { c: Copy; action: Action; fmt: (value: string) => string; setStatus: SetStatus }) { +function MonitorTab({ data, c, action, fmt, setStatus }: { data: Overview; c: Copy; action: Action; fmt: (value: string) => string; setStatus: SetStatus }) { const [monitor, setMonitor] = useState(null); const [url, setUrl] = useState(''); const load = useCallback(() => fetch('/backend/api/super/monitor').then((response) => response.ok ? response.json() : null).then((value) => { if (value) setMonitor(value); }).catch(() => null), []); @@ -624,15 +688,19 @@ function MonitorTab({ c, action, fmt, setStatus }: { c: Copy; action: Action; fm + + +
{c.tabs.monitor}

{c.monitor.events}

{c.monitor.eventsText}

{monitor && !monitor.events.length &&
{c.monitor.noEvents}
}
{monitor?.events.map((event) => (
- {event.level} + {event.severity}
{event.title} + {c.alerts.categories[event.category as AlertCategoryKey] ?? event.category} · {c.alerts.states[event.deliveryState as keyof Copy['alerts']['states']] ?? event.deliveryState} {event.description && {event.description}} {event.fields.map((field) => {field.name}: {field.value})}
@@ -646,6 +714,154 @@ function MonitorTab({ c, action, fmt, setStatus }: { c: Copy; action: Action; fm ); } +/* ------------------------------------------------------------------- alerts */ + +const ALERT_CATEGORY_KEYS = ['availability', 'security', 'errors', 'delivery', 'billing', 'installations', 'usage'] as const; +type AlertCategoryKey = (typeof ALERT_CATEGORY_KEYS)[number]; +type RouteMode = 'status' | 'channel' | 'digest' | 'log'; +type AlertRoute = { mode: RouteMode; guildId: string | null; channelId: string | null; mention: boolean }; +type AlertSettings = { + routes: Record; + quietHours: { enabled: boolean; start: string; end: string }; + digest: { time: string; mode: 'status' | 'channel'; guildId: string | null; channelId: string | null }; + timezone: string; +}; +type Incident = { id: string; key: string; category: string; severity: string; title: string; description: string | null; openedAt: string; resolvedAt: string | null; acknowledgedAt: string | null; acknowledgedBy: string | null; reminders: number }; +type AlertsData = { settings: AlertSettings; incidents: { open: Incident[]; recent: Incident[] }; mentionAvailable: boolean }; + +function useAlerts() { + const [alerts, setAlerts] = useState(null); + const load = useCallback(() => fetch('/backend/api/super/alerts').then((response) => response.ok ? response.json() : null).then((value) => { if (value) setAlerts(value); }).catch(() => null), []); + useEffect(() => { void load(); }, [load]); + return { alerts, load }; +} + +/** Text channels of a server the bot is in, loaded on demand. */ +function useChannels(guildId: string | null) { + const [channels, setChannels] = useState>([]); + useEffect(() => { + if (!guildId) { setChannels([]); return; } + let cancelled = false; + fetch(`/backend/api/super/status-channel/channels?guildId=${guildId}`) + .then((response) => response.ok ? response.json() : []) + .then((rows) => { if (!cancelled) setChannels(rows); }) + .catch(() => { if (!cancelled) setChannels([]); }); + return () => { cancelled = true; }; + }, [guildId]); + return channels; +} + +function ChannelPicker({ data, c, guildId, channelId, onChange }: { data: Overview; c: Copy; guildId: string | null; channelId: string | null; onChange: (guildId: string | null, channelId: string | null) => void }) { + const channels = useChannels(guildId); + return ( +
+ + +
+ ); +} + +function IncidentsCard({ c, action, fmt }: { c: Copy; action: Action; fmt: (value: string) => string }) { + const { alerts, load } = useAlerts(); + useEffect(() => { + const timer = window.setInterval(() => void load(), 30_000); + return () => window.clearInterval(timer); + }, [load]); + const ack = (incident: Incident) => action(`/backend/api/super/incidents/${incident.id}/ack`, { method: 'POST', body: '{}' }, { reload: false }).then(() => void load(), ignore); + const row = (incident: Incident, open: boolean) => ( +
+ {incident.severity} +
+ {incident.title} + {incident.description && {incident.description}} + + {c.alerts.categories[incident.category as AlertCategoryKey] ?? incident.category} · {c.alerts.openedAt} {fmt(incident.openedAt)} + {incident.resolvedAt ? ` · ${c.alerts.resolvedAt} ${fmt(incident.resolvedAt)}` : ''} + {incident.reminders ? ` · ${incident.reminders} ${c.alerts.reminders}` : ''} + {incident.acknowledgedAt ? ` · ${c.alerts.acked} ${incident.acknowledgedBy ?? '—'}` : ''} + +
+ {incident.key} + {open && !incident.acknowledgedAt ? : } +
+ ); + return ( +
+
{c.tabs.monitor}

{c.alerts.incidents}

{c.alerts.incidentsText}

+

{c.alerts.open}

+ {alerts && !alerts.incidents.open.length &&
{c.alerts.none}
} +
{alerts?.incidents.open.map((incident) => row(incident, true))}
+ {alerts && alerts.incidents.recent.length > 0 && <> +

{c.alerts.recent}

+
{alerts.incidents.recent.map((incident) => row(incident, false))}
+ } +
+ ); +} + +function RoutingCard({ data, c, action }: { data: Overview; c: Copy; action: Action }) { + const { alerts } = useAlerts(); + const [draft, setDraft] = useState(null); + useEffect(() => { if (alerts && !draft) setDraft(alerts.settings); }, [alerts, draft]); + if (!draft) return null; + + const setRoute = (category: AlertCategoryKey, patch: Partial) => + setDraft({ ...draft, routes: { ...draft.routes, [category]: { ...draft.routes[category], ...patch } } }); + const save = () => action('/backend/api/super/alerts/settings', { method: 'PUT', body: JSON.stringify(draft) }, { reload: false }) + .then((body) => { if (body.settings) setDraft(body.settings as AlertSettings); }, ignore); + + return ( +
+
{c.tabs.monitor}

{c.alerts.routingTitle}

{c.alerts.routingText}

+
+ {ALERT_CATEGORY_KEYS.map((category) => { + const route = draft.routes[category]; + return ( +
+ {c.alerts.categories[category]} + + {alerts?.mentionAvailable && (route.mode === 'status' || route.mode === 'channel') && ( + + )} + {route.mode === 'channel' && setRoute(category, { guildId, channelId })} />} +
+ ); + })} +
+ +

{c.alerts.quiet}

+

{c.alerts.quietText}

+
+ + + + +
+ +

{c.alerts.digest}

+

{c.alerts.digestText}

+
+ + +
+ {draft.digest.mode === 'channel' && setDraft({ ...draft, digest: { ...draft.digest, guildId, channelId } })} />} + + +
+ ); +} + /* --------------------------------------------------------------------- logs */ type LogLine = { time: string; level: string; msg: string; context: Record | null }; diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index c9cdc61..2ededfb 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -697,6 +697,14 @@ textarea { min-height: 84px; resize: vertical; line-height: 1.5; } .event-row { grid-template-columns: auto minmax(0, 1fr) auto auto !important; align-items: start; } .event-row > div { min-width: 0; display: flex; flex-direction: column; gap: 3px; } .event-row > div span { color: var(--muted); font-size: .8rem; overflow-wrap: anywhere; } +.event-row .event-meta { color: var(--faint); font-size: .74rem; } +.console-subtitle { margin: 20px 0 10px; font-size: .95rem; } +.alert-routes { display: flex; flex-direction: column; border: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; margin-bottom: 8px; } +.alert-route { display: grid; grid-template-columns: 170px minmax(0, 220px) minmax(0, 1fr); gap: 8px 16px; align-items: center; padding: 12px 16px; background: var(--bg-raised); } +.alert-route + .alert-route { border-top: 1px solid var(--line); } +.alert-route select { margin: 0; } +.alert-route .alert-picker { grid-column: 1 / -1; margin: 0; } +@media (max-width: 760px) { .alert-route { grid-template-columns: 1fr; } } .log-lines { display: flex; flex-direction: column; border: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; } .log-line + .log-line { border-top: 1px solid var(--line); } .log-line summary { display: grid; grid-template-columns: auto auto minmax(0, 1fr); gap: 12px; align-items: center; padding: 8px 12px; cursor: pointer; font-size: .82rem; } diff --git a/deploy/runtime/harden-users.sh b/deploy/runtime/harden-users.sh index ce19dae..60af3f4 100755 --- a/deploy/runtime/harden-users.sh +++ b/deploy/runtime/harden-users.sh @@ -23,7 +23,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "PanelAudit", "PanelSession", "PanelRoleBinding", "InstallBlock", "SuperAdminAudit", "InstanceConfig", "StatusChannel", "WaitlistEntry", "InstallGrant", "Coupon", - "InstanceEvent" + "InstanceEvent", "Incident" TO sentinel_api; GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE @@ -32,7 +32,7 @@ TO sentinel_bot; GRANT SELECT ON TABLE "InstallBlock", "InstallGrant", "StatusChannel" TO sentinel_bot; GRANT INSERT ON TABLE "InstanceEvent" TO sentinel_bot; REVOKE ALL ON TABLE "PanelAudit", "PanelSession", "PanelRoleBinding", "SuperAdminAudit", - "InstanceConfig", "WaitlistEntry", "Coupon" FROM sentinel_bot; + "InstanceConfig", "WaitlistEntry", "Coupon", "Incident" FROM sentinel_bot; ALTER ROLE sentinel_api NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION; ALTER ROLE sentinel_bot NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION; SQL diff --git a/packages/db/prisma/migrations/20261021120000_alert_routing/migration.sql b/packages/db/prisma/migrations/20261021120000_alert_routing/migration.sql new file mode 100644 index 0000000..458b69e --- /dev/null +++ b/packages/db/prisma/migrations/20261021120000_alert_routing/migration.sql @@ -0,0 +1,44 @@ +-- Alert routing: categories and severities, delivery state, incidents, +-- routing settings and the daily digest. +ALTER TABLE "InstanceEvent" ADD COLUMN "category" TEXT NOT NULL DEFAULT 'errors'; +ALTER TABLE "InstanceEvent" ADD COLUMN "severity" TEXT NOT NULL DEFAULT 'info'; +ALTER TABLE "InstanceEvent" ADD COLUMN "deliveryState" TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE "InstanceEvent" ADD COLUMN "deliveredAt" TIMESTAMP(3); +-- Events recorded so far were already posted by the bot: never resend them. +UPDATE "InstanceEvent" SET "deliveryState" = 'log'; +CREATE INDEX "InstanceEvent_deliveryState_createdAt_idx" ON "InstanceEvent"("deliveryState", "createdAt"); +-- Instance-wide delivery checks (pending backlog, failures) without a server filter. +CREATE INDEX "LogEvent_dispatchState_createdAt_idx" ON "LogEvent"("dispatchState", "createdAt"); + +ALTER TABLE "InstanceConfig" ADD COLUMN "alertSettings" JSONB NOT NULL DEFAULT '{}'; +ALTER TABLE "InstanceConfig" ADD COLUMN "lastDigestAt" TIMESTAMP(3); + +CREATE TABLE "Incident" ( + "id" TEXT NOT NULL, + "key" TEXT NOT NULL, + "category" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "title" TEXT NOT NULL, + "description" TEXT, + "openedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "resolvedAt" TIMESTAMP(3), + "acknowledgedAt" TIMESTAMP(3), + "acknowledgedBy" TEXT, + "lastRemindedAt" TIMESTAMP(3), + "reminders" INTEGER NOT NULL DEFAULT 0, + + CONSTRAINT "Incident_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "Incident_key_resolvedAt_idx" ON "Incident"("key", "resolvedAt"); +CREATE INDEX "Incident_openedAt_idx" ON "Incident"("openedAt" DESC); + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'sentinel_api') THEN + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "Incident" TO sentinel_api; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'sentinel_bot') THEN + REVOKE ALL ON TABLE "Incident" FROM sentinel_bot; + END IF; +END +$$; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 900ae65..6af9795 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -103,6 +103,7 @@ model LogEvent { @@index([guildId, createdAt(sort: Desc)]) @@index([guildId, eventKey, createdAt(sort: Desc)]) @@index([guildId, dispatchState, createdAt(sort: Desc)]) + @@index([dispatchState, createdAt]) @@index([actorId]) @@index([targetId]) @@index([channelId]) @@ -214,6 +215,9 @@ model InstanceConfig { discordSkus Json @default("{}") // Discord webhook for alarms that must work when the bot is down (encrypted). alertWebhookUrl String? + // Alert routing, quiet hours and daily digest (see apps/api/src/alert-rules.ts). + alertSettings Json @default("{}") + lastDigestAt DateTime? updatedAt DateTime @updatedAt } @@ -281,12 +285,39 @@ model Coupon { // servers added, removed or rejected, quotas, bot down/restored. The bot can // only insert; description and fields are encrypted in `details`. model InstanceEvent { - id String @id @default(cuid()) - source String - level String - title String - details Json - createdAt DateTime @default(now()) + id String @id @default(cuid()) + source String + level String + // Routing: category (where it goes) and severity (how loudly). + category String @default("errors") + severity String @default("info") + title String + details Json + // pending → sent | digest → digested | log | failed + deliveryState String @default("pending") + deliveredAt DateTime? + createdAt DateTime @default(now()) @@index([createdAt(sort: Desc)]) + @@index([deliveryState, createdAt]) +} + +// A problem that lasts: opened and resolved by key (bot.down, discord.sync, +// delivery.backlog, ...), reminded until resolved or acknowledged. +model Incident { + id String @id @default(cuid()) + key String + category String + severity String + title String + description String? + openedAt DateTime @default(now()) + resolvedAt DateTime? + acknowledgedAt DateTime? + acknowledgedBy String? + lastRemindedAt DateTime? + reminders Int @default(0) + + @@index([key, resolvedAt]) + @@index([openedAt(sort: Desc)]) } diff --git a/packages/shared/src/alerts.ts b/packages/shared/src/alerts.ts new file mode 100644 index 0000000..9d59fec --- /dev/null +++ b/packages/shared/src/alerts.ts @@ -0,0 +1,16 @@ +/** + * Instance alerts: every notice has a category, which decides where it goes, + * and a severity, which decides how loudly (mention, quiet hours, reminders). + */ + +export const ALERT_CATEGORIES = ['availability', 'security', 'errors', 'delivery', 'billing', 'installations', 'usage'] as const; +export type AlertCategory = (typeof ALERT_CATEGORIES)[number]; + +export const ALERT_SEVERITIES = ['critical', 'warning', 'info'] as const; +export type AlertSeverity = (typeof ALERT_SEVERITIES)[number]; + +export const isAlertCategory = (value: unknown): value is AlertCategory => + typeof value === 'string' && (ALERT_CATEGORIES as readonly string[]).includes(value); + +export const isAlertSeverity = (value: unknown): value is AlertSeverity => + typeof value === 'string' && (ALERT_SEVERITIES as readonly string[]).includes(value); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index c84f2c5..1cbbd15 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -83,3 +83,4 @@ export const eventDefinition = (key: string) => EVENT_CATALOG.find((event) => ev export * from './plans.js'; export * from './log-buffer.js'; +export * from './alerts.js';