haxurus · Oct 20, 2026 · Oct 20, 2026
@@ -82,13 +82,14 @@ jobs:
           as_bot 'SELECT count(*) FROM "InstallBlock"'
           as_bot 'SELECT count(*) FROM "InstallGrant"'
           as_bot 'SELECT count(*) FROM "StatusChannel"'
-          for table in PanelSession PanelAudit PanelRoleBinding SuperAdminAudit InstanceConfig WaitlistEntry Coupon; do
+          as_bot "INSERT INTO \"InstanceEvent\" (id, source, level, title, details) VALUES ('ci', 'bot', 'info', 'ci', '{}')"
+          for table in PanelSession PanelAudit PanelRoleBinding SuperAdminAudit InstanceConfig WaitlistEntry Coupon InstanceEvent; do
             if as_bot "SELECT count(*) FROM \"$table\"" 2>/dev/null; then echo "sentinel_bot can read $table" >&2; exit 1; fi
           done
           for table in InstallBlock InstallGrant StatusChannel; do
             if as_bot "DELETE FROM \"$table\"" 2>/dev/null; then echo "sentinel_bot can modify $table" >&2; exit 1; fi
           done
-          for table in GuildSettings LogRoute LogEvent MessageSnapshot PanelAudit PanelSession PanelRoleBinding InstallBlock SuperAdminAudit InstanceConfig StatusChannel WaitlistEntry InstallGrant Coupon; do
+          for table in GuildSettings LogRoute LogEvent MessageSnapshot PanelAudit PanelSession PanelRoleBinding InstallBlock SuperAdminAudit InstanceConfig StatusChannel WaitlistEntry InstallGrant Coupon InstanceEvent; do
             as_api "SELECT count(*) FROM \"$table\""
           done
           if as_api 'CREATE TABLE ci_should_fail (id int)' 2>/dev/null; then echo 'sentinel_api can create tables' >&2; exit 1; fi

@@ -201,8 +201,12 @@ The full list of limits lives in [`packages/shared/src/plans.ts`](packages/share
 - public contact details shown on the pricing page;
 - **Discord subscriptions**: the SKU ID of each plan and billing period, the last sync and a manual "sync now";
 - the **status channel**: one server and channel, chosen among those the bot can see, that receives bot startup, errors, warnings, reached quotas and servers added, removed or rejected — these notices are no longer posted in individual servers;
+- **monitoring**: the bot watchdog (checked every 30 seconds by the API) with *down* and *restored* alarms posted to a Discord webhook, which works even when the bot is stopped, and the feed of instance events (startup, errors, servers added, removed or rejected, quotas), kept for 90 days;
+- **logs**: the latest log lines of the bot and the API since their last restart, redacted and kept in memory only; Docker log files are capped at 5 × 10 MB per service;
 - installation blacklist and an audit log of every action, including automatic plan expiries.
 
+For an external uptime monitor, `/backend/health` reports the API and the database and `/backend/health/bot` the bot's connection to Discord (200 or 503, no details).
+
 ## Discord limitations
 
 Sentinel records only what Discord exposes to the bot through the Gateway, REST and Audit Log with the granted intents and permissions. It cannot read private DMs between users, see what someone is viewing, or record voice audio. Attributing an action to a moderator relies on the Audit Log and may be missing when Discord does not provide an entry. Attachments are referenced by their Discord URL, not downloaded.

@@ -37,6 +37,27 @@ function id(value: string) {
   return value;
 }
 
+/**
+ * Bot liveness for the watchdog: null when the process does not answer,
+ * ok=false when it answers but is not connected to Discord.
+ */
+export async function botHealth(): Promise<{ ok: boolean; guilds: number } | null> {
+  try {
+    const response = await fetch(`${internalUrl}/healthz`, { signal: AbortSignal.timeout(5_000) });
+    const body = await response.json().catch(() => ({})) as { ok?: boolean; guilds?: number };
+    return { ok: response.ok && body.ok === true, guilds: Number(body.guilds) || 0 };
+  } catch {
+    return null;
+  }
+}
+
+export type BotLogEntry = { time: string; level: string; msg: string; context: Record<string, unknown> | null };
+
+export async function getBotLogs(level: string, limit: number) {
+  const query = new URLSearchParams({ level, limit: String(limit) });
+  return api<BotLogEntry[]>(`/logs?${query}`);
+}
+
 export async function getGuildResources(guildId: string) {
   return api<{
     channels: Array<{ id: string; name: string; type: number; parent_id?: string | null; position?: number }>;

@@ -5,20 +5,25 @@ import helmet from '@fastify/helmet';
 import rateLimit from '@fastify/rate-limit';
 import { z } from 'zod';
 import { prisma } from '@sentinel/db';
-import { EVENT_CATALOG, tierAllows } from '@sentinel/shared';
+import { EVENT_CATALOG, createLogBuffer, tierAllows } from '@sentinel/shared';
+import pino from 'pino';
+import { registerMonitoringRoutes, startMonitoring } from './monitoring.js';
 import { config } from './config.js';
 import { createSession, destroySession, isSuperAdminUserId, randomToken, requireGuild, requireSession, requireSuperAdmin, resolveGuildAccess, type OAuthGuild } from './auth.js';
 import { getBotGuilds, getGuildResources, leaveBotGuild } from './discord.js';
 import { panelAudit } from './audit.js';
 import { superAdminAudit } from './super-audit.js';
-import { decryptText, unprotectJson } from './security.js';
+import { decryptText, redactText, unprotectJson } from './security.js';
 import { exportJsonChunks, findForeignReference, isValidTimeZone, rateLimitKey } from './helpers.js';
 import { planOverview, registerPlanRoutes } from './plan-routes.js';
 import { guildTier, sweepExpiredPlans } from './plan-service.js';
 import { syncDiscordPlans } from './discord-billing.js';
 import { filterEntryCount, logChannelSet, publicGuildSettings, requiredTierForKey } from './plan-rules.js';
 import { Readable } from 'node:stream';
 
+/** Latest API log lines, shown in the super console. */
+const apiLogs = createLogBuffer(2_000, redactText);
+
 const app = Fastify({
   // Trust exactly one hop (the Next.js proxy in front of the API), as the
   // former `trustProxy: 1`; fastify >= 5.12 only accepts this as a function.
@@ -29,6 +34,11 @@ const app = Fastify({
   maxParamLength: 256,
   logger: {
     level: process.env.LOG_LEVEL ?? 'info',
+    // stdout for Docker, plus the latest lines in memory for the super console.
+    stream: pino.multistream([
+      { level: (process.env.LOG_LEVEL ?? 'info') as pino.Level, stream: process.stdout },
+      { level: (process.env.LOG_LEVEL ?? 'info') as pino.Level, stream: apiLogs }
+    ]),
     redact: {
       paths: ['req.headers.authorization', 'req.headers.cookie', 'res.headers["set-cookie"]', 'client_secret', '*.client_secret', '*.token', '*.password', '*.secret'],
       censor: '[REDACTED]'
@@ -101,8 +111,9 @@ const runDiscordSync = () => syncDiscordPlans(app.log).catch((error) => app.log.
 setTimeout(() => void runDiscordSync(), 30_000).unref();
 setInterval(() => void runDiscordSync(), 2 * 60 * 1000).unref();
 
-app.get('/health/live', async () => ({ ok: true, service: 'api' }));
-app.get('/health', async (_request, reply) => {
+// Health checks run every few seconds: only problems are logged.
+app.get('/health/live', { logLevel: 'warn' }, async () => ({ ok: true, service: 'api' }));
+app.get('/health', { logLevel: 'warn' }, async (_request, reply) => {
   try {
     await prisma.$queryRaw`SELECT 1`;
     return { ok: true, service: 'api', database: 'ready' };
@@ -780,6 +791,8 @@ app.get('/api/guilds/:guildId/export', async (request, reply) => {
 });
 
 await registerPlanRoutes(app);
+await registerMonitoringRoutes(app, apiLogs);
+startMonitoring(app.log);
 
 const shutdown = async (signal: string) => {
   app.log.info({ signal }, 'Shutting down');

@@ -0,0 +1,176 @@
+import type { FastifyInstance } from 'fastify';
+import { z } from 'zod';
+import { prisma } from '@sentinel/db';
+import { LOG_LEVEL_RANK, type LogBuffer, type LogLevelName } from '@sentinel/shared';
+import { requireSuperAdmin } from './auth.js';
+import { botHealth, getBotLogs } from './discord.js';
+import { decryptText, encryptText, protectJson, unprotectJson } from './security.js';
+import { superAdminAudit } from './super-audit.js';
+
+/**
+ * Super console monitoring: the bot watchdog with "down" and "restored"
+ * alarms, the instance event feed and the latest logs of the API and the bot.
+ *
+ * Alarms go to a Discord webhook, not through the bot: they must reach you
+ * precisely when the bot is not running.
+ */
+
+type Logger = { info: (obj: object, msg: string) => void; warn: (obj: object, msg: string) => void };
+type Level = 'info' | 'ok' | 'warn' | 'error';
+
+const CHECK_INTERVAL_MS = 30_000;
+/**
+ * Consecutive failed checks before the bot is declared down (about two
+ * minutes): a deploy restarts the bot in less, without raising an alarm.
+ */
+const FAILURES_BEFORE_DOWN = 4;
+const EVENT_RETENTION_DAYS = 90;
+const WEBHOOK = /^https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/\d{17,20}\/[\w-]{20,}$/;
+const COLORS: Record<Level, number> = { info: 0xa78bfa, ok: 0x3ccf8e, warn: 0xf5a524, error: 0xf2555a };
+
+type BotState = { status: 'unknown' | 'up' | 'down'; since: string; lastCheck: string | null; failures: number; guilds: number | null };
+const state: BotState = { status: 'unknown', since: new Date().toISOString(), lastCheck: null, failures: 0, guilds: null };
+export const botState = () => ({ ...state });
+
+const formatDuration = (ms: number) => {
+  const minutes = Math.round(ms / 60_000);
+  if (minutes < 60) return `${minutes} min`;
+  const hours = Math.floor(minutes / 60);
+  return `${hours} h ${minutes % 60} min`;
+};
+
+export async function recordEvent(level: Level, title: string, description: string | null = null) {
+  await prisma.instanceEvent.create({ data: { source: 'api', level, title, details: protectJson({ description, fields: [] }) } });
+}
+
+async function webhookUrl() {
+  const row = await prisma.instanceConfig.findUnique({ where: { id: 1 }, select: { alertWebhookUrl: true } });
+  return row?.alertWebhookUrl ? decryptText(row.alertWebhookUrl) : null;
+}
+
+async function postWebhook(url: string, level: Level, title: string, description: string) {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      username: 'Sentinel monitor',
+      allowed_mentions: { parse: [] },
+      embeds: [{ title, description, color: COLORS[level], timestamp: new Date().toISOString(), footer: { text: 'Sentinel · monitoraggio' } }]
+    }),
+    signal: AbortSignal.timeout(8_000)
+  });
+  if (!response.ok) throw new Error(`Webhook responded ${response.status}`);
+}
+
+async function alarm(level: Level, title: string, description: string, log: Logger) {
+  await recordEvent(level, title, description).catch((error) => log.warn({ err: error }, 'Unable to record instance event'));
+  const url = await webhookUrl().catch(() => null);
+  if (!url) return;
+  await postWebhook(url, level, title, description).catch((error) => log.warn({ err: error }, 'Alarm webhook failed'));
+}
+
+async function checkBot(log: Logger) {
+  const now = new Date();
+  const health = await botHealth().catch(() => null);
+  state.lastCheck = now.toISOString();
+  if (health?.ok) {
+    state.guilds = health.guilds;
+    state.failures = 0;
+    if (state.status === 'down') {
+      const downtime = formatDuration(now.getTime() - new Date(state.since).getTime());
+      await alarm('ok', 'Bot ripristinato', `Sentinel è di nuovo connesso a Discord dopo ${downtime} di interruzione (${health.guilds} server).`, log);
+    }
+    if (state.status !== 'up') {
+      state.status = 'up';
+      state.since = now.toISOString();
+    }
+    return;
+  }
+  state.failures += 1;
+  if (state.failures >= FAILURES_BEFORE_DOWN && state.status !== 'down') {
+    const wasUp = state.status === 'up';
+    state.status = 'down';
+    state.since = now.toISOString();
+    await alarm('error', 'Bot non raggiungibile', health
+      ? 'Il processo del bot risponde ma non è connesso a Discord: i log non vengono registrati.'
+      : `Il bot non risponde da circa ${Math.round(FAILURES_BEFORE_DOWN * CHECK_INTERVAL_MS / 60_000)} minuti${wasUp ? '' : ' (dall’avvio dell’API)'}: i log non vengono registrati.`, log);
+  }
+}
+
+export function startMonitoring(log: Logger) {
+  setTimeout(() => void checkBot(log), 10_000).unref();
+  setInterval(() => void checkBot(log), CHECK_INTERVAL_MS).unref();
+  const cleanup = () => prisma.instanceEvent.deleteMany({ where: { createdAt: { lt: new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000) } } }).catch(() => null);
+  void cleanup();
+  setInterval(() => void cleanup(), 12 * 60 * 60 * 1000).unref();
+}
+
+export async function registerMonitoringRoutes(app: FastifyInstance, apiLogs: LogBuffer) {
+  // Public and detail-free, for an external uptime monitor.
+  app.get('/health/bot', { logLevel: 'warn' }, async (_request, reply) => {
+    const ok = state.status === 'up';
+    return reply.code(ok ? 200 : 503).send({ ok, service: 'bot' });
+  });
+
+  app.get('/api/super/monitor', async (request, reply) => {
+    const session = await requireSuperAdmin(request, reply);
+    if (!session) return;
+    const [rows, url] = await Promise.all([
+      prisma.instanceEvent.findMany({ orderBy: { createdAt: 'desc' }, take: 200 }),
+      webhookUrl().catch(() => null)
+    ]);
+    const events = rows.map((row) => {
+      let details: { description?: string | null; fields?: Array<{ name: string; value: string }> } = {};
+      try {
+        details = unprotectJson(row.details) as typeof details;
+      } catch {
+        details = { description: null, fields: [] };
+      }
+      return { id: row.id, source: row.source, level: row.level, title: row.title, description: details.description ?? null, fields: details.fields ?? [], createdAt: row.createdAt };
+    });
+    return { bot: botState(), events, webhook: url ? { configured: true, hint: `…${url.slice(-6)}` } : { configured: false, hint: null } };
+  });
+
+  app.get('/api/super/logs', async (request, reply) => {
+    const session = await requireSuperAdmin(request, reply);
+    if (!session) return;
+    const parsed = z.object({
+      service: z.enum(['api', 'bot']).default('bot'),
+      level: z.enum(Object.keys(LOG_LEVEL_RANK) as [LogLevelName, ...LogLevelName[]]).default('info'),
+      limit: z.coerce.number().int().min(1).max(1_000).default(300)
+    }).safeParse(request.query);
+    if (!parsed.success) return reply.code(400).send({ error: 'INVALID_QUERY' });
+    const { service, level, limit } = parsed.data;
+    if (service === 'api') return { service, entries: apiLogs.list(level, limit) };
+    try {
+      return { service, entries: await getBotLogs(level, limit) };
+    } catch {
+      return reply.code(502).send({ error: 'BOT_UNAVAILABLE' });
+    }
+  });
+
+  app.put('/api/super/alert-webhook', async (request, reply) => {
+    const session = await requireSuperAdmin(request, reply);
+    if (!session) return;
+    const parsed = z.object({ url: z.union([z.literal(''), z.string().trim().regex(WEBHOOK)]).nullable() }).safeParse(request.body);
+    if (!parsed.success) return reply.code(400).send({ error: 'INVALID_WEBHOOK' });
+    const url = parsed.data.url || null;
+    const stored = url ? encryptText(url) : null;
+    await prisma.instanceConfig.upsert({ where: { id: 1 }, update: { alertWebhookUrl: stored }, create: { id: 1, alertWebhookUrl: stored } });
+    await superAdminAudit(request, session, url ? 'monitor.webhook_set' : 'monitor.webhook_removed', null, null);
+    return { ok: true, configured: Boolean(url) };
+  });
+
+  app.post('/api/super/alert-webhook/test', async (request, reply) => {
+    const session = await requireSuperAdmin(request, reply);
+    if (!session) return;
+    const url = await webhookUrl().catch(() => null);
+    if (!url) return reply.code(400).send({ error: 'WEBHOOK_NOT_SET' });
+    try {
+      await postWebhook(url, 'info', 'Allarmi attivi', 'Messaggio di prova dalla super console: qui arriveranno gli avvisi quando il bot non è raggiungibile e quando torna attivo.');
+    } catch {
+      return reply.code(502).send({ error: 'WEBHOOK_FAILED' });
+    }
+    return { ok: true };
+  });
+}
@@ -78,7 +78,8 @@ export async function registerPlanRoutes(app: FastifyInstance) {
     };
     const config = await prisma.instanceConfig.upsert({ where: { id: 1 }, update: data, create: { id: 1, ...data } });
     await superAdminAudit(request, session, 'instance.config', null, null, data);
-    return config;
+    const { alertWebhookUrl: _webhook, ...visible } = config;
+    return visible;
   });
 
   // ------------------------------------------ super: Discord subscriptions
@@ -261,7 +262,7 @@ export async function planOverview(botGuildIds: string[]) {
       select: { guildId: true, planTier: true, planBilling: true, planExpiresAt: true, planSource: true, planNote: true, planUpdatedAt: true }
     })
   ]);
-  const { discordSkus, ...settings } = config;
+  const { discordSkus, alertWebhookUrl: _webhook, ...settings } = config;
   return {
     config: settings,
     statusChannel,

@@ -0,0 +1,20 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { createLogBuffer } from '@sentinel/shared';
+
+test('log buffer keeps the latest redacted lines, newest first, by level', () => {
+  const buffer = createLogBuffer(3, (value) => value.replace(/secret-\w+/g, '[REDACTED]'));
+  buffer.write(`${JSON.stringify({ level: 30, time: 1, msg: 'first', pid: 1, hostname: 'x' })}\n`);
+  buffer.write(`${JSON.stringify({ level: 40, time: 2, msg: 'token secret-abc', guildId: '1' })}\n${JSON.stringify({ level: 50, time: 3, msg: 'boom', error: 'secret-def' })}\n`);
+  buffer.write('not json');
+
+  const all = buffer.list('trace', 10);
+  assert.equal(all.length, 3, 'capacity is respected');
+  assert.deepEqual(all.map((entry) => entry.msg), ['not json', 'boom', 'token [REDACTED]']);
+  assert.deepEqual(all[1]!.context, { error: '[REDACTED]' });
+  assert.equal(all[2]!.context?.guildId, '1');
+  assert.equal(all[2]!.context?.pid, undefined, 'process noise is dropped');
+
+  assert.deepEqual(buffer.list('warn', 10).map((entry) => entry.level), ['error', 'warn']);
+  assert.equal(buffer.list('trace', 1).length, 1);
+});
@@ -3,6 +3,8 @@ import http from 'node:http';
 import { DiscordAPIError, Routes, type Client } from 'discord.js';
 import { editOwnGuildProfile, leaveGuild } from './discord-actions.js';
 import type { StatusNotifier } from './status.js';
+import { LOG_LEVEL_RANK, type LogLevelName } from '@sentinel/shared';
+import { logBuffer } from './logger.js';
 
 const SNOWFLAKE = /^\d{17,20}$/;
 
@@ -108,6 +110,14 @@ export function startInternalApi(client: Client, secret: string, port = 3002, st
         return;
       }
 
+      if (req.method === 'GET' && url.pathname === '/logs') {
+        const requested = url.searchParams.get('level') ?? 'info';
+        const level = (requested in LOG_LEVEL_RANK ? requested : 'info') as LogLevelName;
+        const limit = Math.min(1_000, Math.max(1, Number(url.searchParams.get('limit')) || 500));
+        res.end(JSON.stringify(logBuffer.list(level, limit)));
+        return;
+      }
+
       if (req.method === 'GET' && url.pathname === '/entitlements') {
         res.end(JSON.stringify(await activeEntitlements(client)));
         return;

@@ -1,4 +1,13 @@
 import pino from 'pino';
+import { createLogBuffer } from '@sentinel/shared';
 import { config } from './config.js';
+import { redactText } from './security.js';
 
-export const logger = pino({ level: config.logLevel });
+/** Latest log lines, readable from the super console through the internal API. */
+export const logBuffer = createLogBuffer(2_000, redactText);
+
+const level = config.logLevel as pino.Level;
+export const logger = pino({ level }, pino.multistream([
+  { level, stream: process.stdout },
+  { level, stream: logBuffer }
+]));