From 2a91344919c79e01d6bb868b6d66624fa158d35d Mon Sep 17 00:00:00 2001 From: Haxurus <72351758+haxurus@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:03:15 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20super=20console=20monitoring=20?= =?UTF-8?q?=E2=80=94=20bot=20watchdog=20alarms,=20instance=20events,=20log?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - API watchdog checks the bot every 30 s; after ~2 minutes without answer it posts "bot unreachable" to a Discord webhook set in the super console (stored encrypted), and "bot restored" with the downtime when it is back. - InstanceEvent table: every status notice of the bot (startup, errors, servers added/removed/rejected, quotas) plus watchdog alarms, shown in the super console and kept 90 days. The bot can only insert. - Logs tab: latest bot and API log lines from in-memory, redacted buffers. Health-check requests are no longer logged at info level. - Docker json-file logs capped at 5 x 10 MB per service. - Public /health/bot for external uptime monitors. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 5 +- README.md | 4 + apps/api/src/discord.ts | 21 ++ apps/api/src/index.ts | 21 +- apps/api/src/monitoring.ts | 176 +++++++++++++++ apps/api/src/plan-routes.ts | 5 +- apps/api/test/log-buffer.test.ts | 20 ++ apps/bot/src/internal-api.ts | 10 + apps/bot/src/logger.ts | 11 +- apps/bot/src/status.ts | 18 +- apps/web/app/_components/SuperConsole.tsx | 203 +++++++++++++++++- apps/web/app/globals.css | 18 ++ deploy/docker-compose.prod.yml | 15 ++ deploy/runtime/harden-users.sh | 4 +- docker-compose.yml | 15 ++ docs/DEPLOYMENT.md | 7 + .../migration.sql | 30 +++ packages/db/prisma/schema.prisma | 16 ++ packages/shared/src/index.ts | 1 + packages/shared/src/log-buffer.ts | 63 ++++++ 20 files changed, 648 insertions(+), 15 deletions(-) create mode 100644 apps/api/src/monitoring.ts create mode 100644 apps/api/test/log-buffer.test.ts create mode 100644 packages/db/prisma/migrations/20261020120000_instance_monitoring/migration.sql create mode 100644 packages/shared/src/log-buffer.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e32cdcc..1bf9ab5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,13 +82,14 @@ jobs: as_bot 'SELECT count(*) FROM "InstallBlock"' as_bot 'SELECT count(*) FROM "InstallGrant"' as_bot 'SELECT count(*) FROM "StatusChannel"' - for table in PanelSession PanelAudit PanelRoleBinding SuperAdminAudit InstanceConfig WaitlistEntry Coupon; do + as_bot "INSERT INTO \"InstanceEvent\" (id, source, level, title, details) VALUES ('ci', 'bot', 'info', 'ci', '{}')" + for table in PanelSession PanelAudit PanelRoleBinding SuperAdminAudit InstanceConfig WaitlistEntry Coupon InstanceEvent; do if as_bot "SELECT count(*) FROM \"$table\"" 2>/dev/null; then echo "sentinel_bot can read $table" >&2; exit 1; fi done for table in InstallBlock InstallGrant StatusChannel; do if as_bot "DELETE FROM \"$table\"" 2>/dev/null; then echo "sentinel_bot can modify $table" >&2; exit 1; fi done - for table in GuildSettings LogRoute LogEvent MessageSnapshot PanelAudit PanelSession PanelRoleBinding InstallBlock SuperAdminAudit InstanceConfig StatusChannel WaitlistEntry InstallGrant Coupon; do + for table in GuildSettings LogRoute LogEvent MessageSnapshot PanelAudit PanelSession PanelRoleBinding InstallBlock SuperAdminAudit InstanceConfig StatusChannel WaitlistEntry InstallGrant Coupon InstanceEvent; do as_api "SELECT count(*) FROM \"$table\"" done if as_api 'CREATE TABLE ci_should_fail (id int)' 2>/dev/null; then echo 'sentinel_api can create tables' >&2; exit 1; fi diff --git a/README.md b/README.md index b592f54..8083b66 100644 --- a/README.md +++ b/README.md @@ -201,8 +201,12 @@ The full list of limits lives in [`packages/shared/src/plans.ts`](packages/share - public contact details shown on the pricing page; - **Discord subscriptions**: the SKU ID of each plan and billing period, the last sync and a manual "sync now"; - the **status channel**: one server and channel, chosen among those the bot can see, that receives bot startup, errors, warnings, reached quotas and servers added, removed or rejected — these notices are no longer posted in individual servers; +- **monitoring**: the bot watchdog (checked every 30 seconds by the API) with *down* and *restored* alarms posted to a Discord webhook, which works even when the bot is stopped, and the feed of instance events (startup, errors, servers added, removed or rejected, quotas), kept for 90 days; +- **logs**: the latest log lines of the bot and the API since their last restart, redacted and kept in memory only; Docker log files are capped at 5 × 10 MB per service; - installation blacklist and an audit log of every action, including automatic plan expiries. +For an external uptime monitor, `/backend/health` reports the API and the database and `/backend/health/bot` the bot's connection to Discord (200 or 503, no details). + ## Discord limitations Sentinel records only what Discord exposes to the bot through the Gateway, REST and Audit Log with the granted intents and permissions. It cannot read private DMs between users, see what someone is viewing, or record voice audio. Attributing an action to a moderator relies on the Audit Log and may be missing when Discord does not provide an entry. Attachments are referenced by their Discord URL, not downloaded. diff --git a/apps/api/src/discord.ts b/apps/api/src/discord.ts index 95a0aa1..be06dc9 100644 --- a/apps/api/src/discord.ts +++ b/apps/api/src/discord.ts @@ -37,6 +37,27 @@ function id(value: string) { return value; } +/** + * Bot liveness for the watchdog: null when the process does not answer, + * ok=false when it answers but is not connected to Discord. + */ +export async function botHealth(): Promise<{ ok: boolean; guilds: number } | null> { + try { + const response = await fetch(`${internalUrl}/healthz`, { signal: AbortSignal.timeout(5_000) }); + const body = await response.json().catch(() => ({})) as { ok?: boolean; guilds?: number }; + return { ok: response.ok && body.ok === true, guilds: Number(body.guilds) || 0 }; + } catch { + return null; + } +} + +export type BotLogEntry = { time: string; level: string; msg: string; context: Record | null }; + +export async function getBotLogs(level: string, limit: number) { + const query = new URLSearchParams({ level, limit: String(limit) }); + return api(`/logs?${query}`); +} + export async function getGuildResources(guildId: string) { return api<{ channels: Array<{ id: string; name: string; type: number; parent_id?: string | null; position?: number }>; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 0206587..b765388 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -5,13 +5,15 @@ import helmet from '@fastify/helmet'; import rateLimit from '@fastify/rate-limit'; import { z } from 'zod'; import { prisma } from '@sentinel/db'; -import { EVENT_CATALOG, tierAllows } from '@sentinel/shared'; +import { EVENT_CATALOG, createLogBuffer, tierAllows } from '@sentinel/shared'; +import pino from 'pino'; +import { registerMonitoringRoutes, startMonitoring } from './monitoring.js'; import { config } from './config.js'; import { createSession, destroySession, isSuperAdminUserId, randomToken, requireGuild, requireSession, requireSuperAdmin, resolveGuildAccess, type OAuthGuild } from './auth.js'; import { getBotGuilds, getGuildResources, leaveBotGuild } from './discord.js'; import { panelAudit } from './audit.js'; import { superAdminAudit } from './super-audit.js'; -import { decryptText, unprotectJson } from './security.js'; +import { decryptText, redactText, unprotectJson } from './security.js'; import { exportJsonChunks, findForeignReference, isValidTimeZone, rateLimitKey } from './helpers.js'; import { planOverview, registerPlanRoutes } from './plan-routes.js'; import { guildTier, sweepExpiredPlans } from './plan-service.js'; @@ -19,6 +21,9 @@ import { syncDiscordPlans } from './discord-billing.js'; import { filterEntryCount, logChannelSet, publicGuildSettings, requiredTierForKey } from './plan-rules.js'; import { Readable } from 'node:stream'; +/** Latest API log lines, shown in the super console. */ +const apiLogs = createLogBuffer(2_000, redactText); + const app = Fastify({ // Trust exactly one hop (the Next.js proxy in front of the API), as the // former `trustProxy: 1`; fastify >= 5.12 only accepts this as a function. @@ -29,6 +34,11 @@ const app = Fastify({ maxParamLength: 256, logger: { level: process.env.LOG_LEVEL ?? 'info', + // stdout for Docker, plus the latest lines in memory for the super console. + stream: pino.multistream([ + { level: (process.env.LOG_LEVEL ?? 'info') as pino.Level, stream: process.stdout }, + { level: (process.env.LOG_LEVEL ?? 'info') as pino.Level, stream: apiLogs } + ]), redact: { paths: ['req.headers.authorization', 'req.headers.cookie', 'res.headers["set-cookie"]', 'client_secret', '*.client_secret', '*.token', '*.password', '*.secret'], censor: '[REDACTED]' @@ -101,8 +111,9 @@ const runDiscordSync = () => syncDiscordPlans(app.log).catch((error) => app.log. setTimeout(() => void runDiscordSync(), 30_000).unref(); setInterval(() => void runDiscordSync(), 2 * 60 * 1000).unref(); -app.get('/health/live', async () => ({ ok: true, service: 'api' })); -app.get('/health', async (_request, reply) => { +// Health checks run every few seconds: only problems are logged. +app.get('/health/live', { logLevel: 'warn' }, async () => ({ ok: true, service: 'api' })); +app.get('/health', { logLevel: 'warn' }, async (_request, reply) => { try { await prisma.$queryRaw`SELECT 1`; return { ok: true, service: 'api', database: 'ready' }; @@ -780,6 +791,8 @@ app.get('/api/guilds/:guildId/export', async (request, reply) => { }); await registerPlanRoutes(app); +await registerMonitoringRoutes(app, apiLogs); +startMonitoring(app.log); const shutdown = async (signal: string) => { app.log.info({ signal }, 'Shutting down'); diff --git a/apps/api/src/monitoring.ts b/apps/api/src/monitoring.ts new file mode 100644 index 0000000..33b7702 --- /dev/null +++ b/apps/api/src/monitoring.ts @@ -0,0 +1,176 @@ +import type { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { prisma } from '@sentinel/db'; +import { LOG_LEVEL_RANK, type LogBuffer, type LogLevelName } from '@sentinel/shared'; +import { requireSuperAdmin } from './auth.js'; +import { botHealth, getBotLogs } from './discord.js'; +import { decryptText, encryptText, protectJson, unprotectJson } from './security.js'; +import { superAdminAudit } from './super-audit.js'; + +/** + * Super console monitoring: the bot watchdog with "down" and "restored" + * alarms, the instance event feed and the latest logs of the API and the bot. + * + * Alarms go to a Discord webhook, not through the bot: they must reach you + * precisely when the bot is not running. + */ + +type Logger = { info: (obj: object, msg: string) => void; warn: (obj: object, msg: string) => void }; +type Level = 'info' | 'ok' | 'warn' | 'error'; + +const CHECK_INTERVAL_MS = 30_000; +/** + * Consecutive failed checks before the bot is declared down (about two + * minutes): a deploy restarts the bot in less, without raising an alarm. + */ +const FAILURES_BEFORE_DOWN = 4; +const EVENT_RETENTION_DAYS = 90; +const WEBHOOK = /^https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/\d{17,20}\/[\w-]{20,}$/; +const COLORS: Record = { info: 0xa78bfa, ok: 0x3ccf8e, warn: 0xf5a524, error: 0xf2555a }; + +type BotState = { status: 'unknown' | 'up' | 'down'; since: string; lastCheck: string | null; failures: number; guilds: number | null }; +const state: BotState = { status: 'unknown', since: new Date().toISOString(), lastCheck: null, failures: 0, guilds: null }; +export const botState = () => ({ ...state }); + +const formatDuration = (ms: number) => { + const minutes = Math.round(ms / 60_000); + if (minutes < 60) return `${minutes} min`; + const hours = Math.floor(minutes / 60); + return `${hours} h ${minutes % 60} min`; +}; + +export async function recordEvent(level: Level, title: string, description: string | null = null) { + await prisma.instanceEvent.create({ data: { source: 'api', level, title, details: protectJson({ description, fields: [] }) } }); +} + +async function webhookUrl() { + const row = await prisma.instanceConfig.findUnique({ where: { id: 1 }, select: { alertWebhookUrl: true } }); + return row?.alertWebhookUrl ? decryptText(row.alertWebhookUrl) : null; +} + +async function postWebhook(url: string, level: Level, title: string, description: string) { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + username: 'Sentinel monitor', + allowed_mentions: { parse: [] }, + embeds: [{ title, description, color: COLORS[level], timestamp: new Date().toISOString(), footer: { text: 'Sentinel · monitoraggio' } }] + }), + signal: AbortSignal.timeout(8_000) + }); + if (!response.ok) throw new Error(`Webhook responded ${response.status}`); +} + +async function alarm(level: Level, title: string, description: string, log: Logger) { + await recordEvent(level, title, description).catch((error) => log.warn({ err: error }, 'Unable to record instance event')); + const url = await webhookUrl().catch(() => null); + if (!url) return; + await postWebhook(url, level, title, description).catch((error) => log.warn({ err: error }, 'Alarm webhook failed')); +} + +async function checkBot(log: Logger) { + const now = new Date(); + const health = await botHealth().catch(() => null); + state.lastCheck = now.toISOString(); + if (health?.ok) { + state.guilds = health.guilds; + state.failures = 0; + if (state.status === 'down') { + const downtime = formatDuration(now.getTime() - new Date(state.since).getTime()); + await alarm('ok', 'Bot ripristinato', `Sentinel è di nuovo connesso a Discord dopo ${downtime} di interruzione (${health.guilds} server).`, log); + } + if (state.status !== 'up') { + state.status = 'up'; + state.since = now.toISOString(); + } + return; + } + state.failures += 1; + if (state.failures >= FAILURES_BEFORE_DOWN && state.status !== 'down') { + const wasUp = state.status === 'up'; + state.status = 'down'; + state.since = now.toISOString(); + await alarm('error', 'Bot non raggiungibile', health + ? 'Il processo del bot risponde ma non è connesso a Discord: i log non vengono registrati.' + : `Il bot non risponde da circa ${Math.round(FAILURES_BEFORE_DOWN * CHECK_INTERVAL_MS / 60_000)} minuti${wasUp ? '' : ' (dall’avvio dell’API)'}: i log non vengono registrati.`, log); + } +} + +export function startMonitoring(log: Logger) { + setTimeout(() => void checkBot(log), 10_000).unref(); + setInterval(() => void checkBot(log), CHECK_INTERVAL_MS).unref(); + const cleanup = () => prisma.instanceEvent.deleteMany({ where: { createdAt: { lt: new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000) } } }).catch(() => null); + void cleanup(); + setInterval(() => void cleanup(), 12 * 60 * 60 * 1000).unref(); +} + +export async function registerMonitoringRoutes(app: FastifyInstance, apiLogs: LogBuffer) { + // Public and detail-free, for an external uptime monitor. + app.get('/health/bot', { logLevel: 'warn' }, async (_request, reply) => { + const ok = state.status === 'up'; + return reply.code(ok ? 200 : 503).send({ ok, service: 'bot' }); + }); + + app.get('/api/super/monitor', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const [rows, url] = await Promise.all([ + prisma.instanceEvent.findMany({ orderBy: { createdAt: 'desc' }, take: 200 }), + webhookUrl().catch(() => null) + ]); + const events = rows.map((row) => { + let details: { description?: string | null; fields?: Array<{ name: string; value: string }> } = {}; + try { + details = unprotectJson(row.details) as typeof details; + } catch { + details = { description: null, fields: [] }; + } + return { id: row.id, source: row.source, level: row.level, title: row.title, description: details.description ?? null, fields: details.fields ?? [], createdAt: row.createdAt }; + }); + return { bot: botState(), events, webhook: url ? { configured: true, hint: `…${url.slice(-6)}` } : { configured: false, hint: null } }; + }); + + app.get('/api/super/logs', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const parsed = z.object({ + service: z.enum(['api', 'bot']).default('bot'), + level: z.enum(Object.keys(LOG_LEVEL_RANK) as [LogLevelName, ...LogLevelName[]]).default('info'), + limit: z.coerce.number().int().min(1).max(1_000).default(300) + }).safeParse(request.query); + if (!parsed.success) return reply.code(400).send({ error: 'INVALID_QUERY' }); + const { service, level, limit } = parsed.data; + if (service === 'api') return { service, entries: apiLogs.list(level, limit) }; + try { + return { service, entries: await getBotLogs(level, limit) }; + } catch { + return reply.code(502).send({ error: 'BOT_UNAVAILABLE' }); + } + }); + + app.put('/api/super/alert-webhook', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const parsed = z.object({ url: z.union([z.literal(''), z.string().trim().regex(WEBHOOK)]).nullable() }).safeParse(request.body); + if (!parsed.success) return reply.code(400).send({ error: 'INVALID_WEBHOOK' }); + const url = parsed.data.url || null; + const stored = url ? encryptText(url) : null; + await prisma.instanceConfig.upsert({ where: { id: 1 }, update: { alertWebhookUrl: stored }, create: { id: 1, alertWebhookUrl: stored } }); + await superAdminAudit(request, session, url ? 'monitor.webhook_set' : 'monitor.webhook_removed', null, null); + return { ok: true, configured: Boolean(url) }; + }); + + app.post('/api/super/alert-webhook/test', async (request, reply) => { + const session = await requireSuperAdmin(request, reply); + if (!session) return; + const url = await webhookUrl().catch(() => null); + if (!url) return reply.code(400).send({ error: 'WEBHOOK_NOT_SET' }); + try { + await postWebhook(url, 'info', 'Allarmi attivi', 'Messaggio di prova dalla super console: qui arriveranno gli avvisi quando il bot non è raggiungibile e quando torna attivo.'); + } catch { + return reply.code(502).send({ error: 'WEBHOOK_FAILED' }); + } + return { ok: true }; + }); +} diff --git a/apps/api/src/plan-routes.ts b/apps/api/src/plan-routes.ts index abf6074..32d283f 100644 --- a/apps/api/src/plan-routes.ts +++ b/apps/api/src/plan-routes.ts @@ -78,7 +78,8 @@ export async function registerPlanRoutes(app: FastifyInstance) { }; const config = await prisma.instanceConfig.upsert({ where: { id: 1 }, update: data, create: { id: 1, ...data } }); await superAdminAudit(request, session, 'instance.config', null, null, data); - return config; + const { alertWebhookUrl: _webhook, ...visible } = config; + return visible; }); // ------------------------------------------ super: Discord subscriptions @@ -261,7 +262,7 @@ export async function planOverview(botGuildIds: string[]) { select: { guildId: true, planTier: true, planBilling: true, planExpiresAt: true, planSource: true, planNote: true, planUpdatedAt: true } }) ]); - const { discordSkus, ...settings } = config; + const { discordSkus, alertWebhookUrl: _webhook, ...settings } = config; return { config: settings, statusChannel, diff --git a/apps/api/test/log-buffer.test.ts b/apps/api/test/log-buffer.test.ts new file mode 100644 index 0000000..c53b2e2 --- /dev/null +++ b/apps/api/test/log-buffer.test.ts @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createLogBuffer } from '@sentinel/shared'; + +test('log buffer keeps the latest redacted lines, newest first, by level', () => { + const buffer = createLogBuffer(3, (value) => value.replace(/secret-\w+/g, '[REDACTED]')); + buffer.write(`${JSON.stringify({ level: 30, time: 1, msg: 'first', pid: 1, hostname: 'x' })}\n`); + buffer.write(`${JSON.stringify({ level: 40, time: 2, msg: 'token secret-abc', guildId: '1' })}\n${JSON.stringify({ level: 50, time: 3, msg: 'boom', error: 'secret-def' })}\n`); + buffer.write('not json'); + + const all = buffer.list('trace', 10); + assert.equal(all.length, 3, 'capacity is respected'); + assert.deepEqual(all.map((entry) => entry.msg), ['not json', 'boom', 'token [REDACTED]']); + assert.deepEqual(all[1]!.context, { error: '[REDACTED]' }); + assert.equal(all[2]!.context?.guildId, '1'); + assert.equal(all[2]!.context?.pid, undefined, 'process noise is dropped'); + + assert.deepEqual(buffer.list('warn', 10).map((entry) => entry.level), ['error', 'warn']); + assert.equal(buffer.list('trace', 1).length, 1); +}); diff --git a/apps/bot/src/internal-api.ts b/apps/bot/src/internal-api.ts index 7860f81..473e762 100644 --- a/apps/bot/src/internal-api.ts +++ b/apps/bot/src/internal-api.ts @@ -3,6 +3,8 @@ import http from 'node:http'; import { DiscordAPIError, Routes, type Client } from 'discord.js'; import { editOwnGuildProfile, leaveGuild } from './discord-actions.js'; import type { StatusNotifier } from './status.js'; +import { LOG_LEVEL_RANK, type LogLevelName } from '@sentinel/shared'; +import { logBuffer } from './logger.js'; const SNOWFLAKE = /^\d{17,20}$/; @@ -108,6 +110,14 @@ export function startInternalApi(client: Client, secret: string, port = 3002, st return; } + if (req.method === 'GET' && url.pathname === '/logs') { + const requested = url.searchParams.get('level') ?? 'info'; + const level = (requested in LOG_LEVEL_RANK ? requested : 'info') as LogLevelName; + const limit = Math.min(1_000, Math.max(1, Number(url.searchParams.get('limit')) || 500)); + res.end(JSON.stringify(logBuffer.list(level, limit))); + return; + } + if (req.method === 'GET' && url.pathname === '/entitlements') { res.end(JSON.stringify(await activeEntitlements(client))); return; diff --git a/apps/bot/src/logger.ts b/apps/bot/src/logger.ts index 99cb818..d31ac22 100644 --- a/apps/bot/src/logger.ts +++ b/apps/bot/src/logger.ts @@ -1,4 +1,13 @@ import pino from 'pino'; +import { createLogBuffer } from '@sentinel/shared'; import { config } from './config.js'; +import { redactText } from './security.js'; -export const logger = pino({ level: config.logLevel }); +/** Latest log lines, readable from the super console through the internal API. */ +export const logBuffer = createLogBuffer(2_000, redactText); + +const level = config.logLevel as pino.Level; +export const logger = pino({ level }, pino.multistream([ + { level, stream: process.stdout }, + { level, stream: logBuffer } +])); diff --git a/apps/bot/src/status.ts b/apps/bot/src/status.ts index 15dfa9f..a3b539e 100644 --- a/apps/bot/src/status.ts +++ b/apps/bot/src/status.ts @@ -3,7 +3,7 @@ import { prisma } from '@sentinel/db'; import { sendStatusMessage } from './dispatcher.js'; import { truncate } from './embed-format.js'; import { logger } from './logger.js'; -import { redactText } from './security.js'; +import { protectJson, redactText } from './security.js'; export type StatusLevel = 'info' | 'ok' | 'warn' | 'error'; @@ -68,7 +68,22 @@ export function createStatusNotifier(client: Client): StatusNotifier { return windowCount > BURST_MAX; }; + /** + * Every notice also goes to the super console's event feed, even when no + * status channel is set. createMany: the bot may insert but not read back. + */ + const record = async (notice: StatusNotice) => { + const description = notice.description ? redactText(notice.description).slice(0, 4000) : null; + const fields = (notice.fields ?? []).filter((field) => field.value).slice(0, 20) + .map((field) => ({ name: field.name.slice(0, 256), value: redactText(field.value).slice(0, 1024) })); + await prisma.instanceEvent.createMany({ + data: [{ source: 'bot', level: notice.level, title: notice.title.slice(0, 256), details: protectJson({ description, fields }) }] + }).catch((error: unknown) => logger.warn({ error: redactText(String(error)) }, 'Unable to record instance event')); + }; + const deliver: StatusNotifier['deliver'] = async (notice, options = {}) => { + if (!options.force && throttled(notice)) return { ok: false, error: 'THROTTLED' }; + if (!options.force) await record(notice); let config: Target; try { config = await target(); @@ -76,7 +91,6 @@ export function createStatusNotifier(client: Client): StatusNotifier { return { ok: false, error: 'STATUS_CONFIG_UNAVAILABLE' }; } if (!config) return { ok: false, error: 'STATUS_CHANNEL_NOT_SET' }; - if (!options.force && throttled(notice)) return { ok: false, error: 'THROTTLED' }; const embed = new EmbedBuilder() .setTitle(truncate(notice.title, 256)) diff --git a/apps/web/app/_components/SuperConsole.tsx b/apps/web/app/_components/SuperConsole.tsx index 1abbd95..d7e03b3 100644 --- a/apps/web/app/_components/SuperConsole.tsx +++ b/apps/web/app/_components/SuperConsole.tsx @@ -44,7 +44,7 @@ type Overview = { planCatalog: Plan[]; discord: DiscordOverview; }; -type Tab = 'servers' | 'contacts' | 'discord' | 'status' | 'blacklist' | 'audit'; +type Tab = 'servers' | 'monitor' | 'logs' | 'contacts' | 'discord' | 'status' | 'blacklist' | 'audit'; const copy = { it: { @@ -53,7 +53,7 @@ const copy = { intro: 'Area riservata al proprietario dell’istanza. Le azioni qui sotto hanno effetto su tutti i server collegati al bot.', dashboard: 'Dashboard', pricing: 'Prezzi', - tabs: { servers: 'Server e piani', contacts: 'Contatti', discord: 'Abbonamenti Discord', status: 'Notifiche', blacklist: 'Blacklist', audit: 'Audit' }, + tabs: { servers: 'Server e piani', monitor: 'Monitoraggio', logs: 'Log', contacts: 'Contatti', discord: 'Abbonamenti Discord', status: 'Notifiche', blacklist: 'Blacklist', audit: 'Audit' }, liveServers: 'Server collegati', liveServersText: 'Elenco live dalla sessione Discord del bot. Da qui assegni il piano di ogni server.', members: 'membri', @@ -88,6 +88,38 @@ const copy = { url: 'Link: server Discord, sito o modulo (https)', save: 'Salva recapiti' }, + monitor: { + title: 'Stato e allarmi', + text: 'L’API controlla il bot ogni 30 secondi. Se non risponde per circa due minuti invia un allarme al webhook qui sotto, e un altro quando torna attivo. Gli avvisi passano da un webhook perché devono arrivare proprio quando il bot è fermo.', + bot: 'Bot', + up: 'Connesso', + down: 'Non raggiungibile', + unknown: 'In verifica', + since: 'dal', + lastCheck: 'ultimo controllo', + servers: 'server', + webhook: 'Webhook Discord per gli allarmi', + webhookHelp: 'In un canale privato: Modifica canale → Integrazioni → Webhook → Nuovo webhook → Copia URL.', + webhookSet: 'Webhook configurato', + webhookNotSet: 'Nessun webhook: gli allarmi restano solo nel registro eventi.', + save: 'Salva webhook', + remove: 'Rimuovi', + test: 'Invia prova', + testOk: 'Messaggio di prova inviato.', + events: 'Eventi dell’istanza', + eventsText: 'Avvio e errori del bot, server aggiunti, rimossi o rifiutati, quote raggiunte, bot giù e ripristinato. Conservati 90 giorni.', + noEvents: 'Nessun evento registrato.', + refresh: 'Aggiorna' + }, + logs: { + title: 'Log recenti', + text: 'Le ultime righe di log di bot e API dall’ultimo riavvio, già ripulite da token e segreti. Restano solo in memoria; i file di Docker sono limitati a 50 MB per servizio.', + service: 'Servizio', + level: 'Livello minimo', + empty: 'Nessuna riga per questo filtro.', + refresh: 'Aggiorna', + auto: 'Aggiornamento automatico' + }, discord: { title: 'Abbonamenti Discord', text: 'I piani a pagamento si vendono solo come abbonamenti per server tramite le App premium di Discord, agli stessi prezzi della pagina prezzi. Crea gli SKU nel Portale sviluppatori (Monetizzazione) e incolla qui i loro ID: i piani dei server si aggiornano da soli ogni due minuti.', @@ -143,6 +175,9 @@ const copy = { guild: 'Server', errors: { INVALID_SKU: 'ID SKU non valido (17-20 cifre).', + INVALID_WEBHOOK: 'URL del webhook non valido: copialo da Integrazioni → Webhook.', + WEBHOOK_NOT_SET: 'Nessun webhook configurato.', + WEBHOOK_FAILED: 'Discord ha rifiutato il messaggio: controlla che il webhook esista ancora.', DUPLICATE_SKU: 'Lo stesso SKU non può valere per due piani.', DISCORD_SYNC_FAILED: 'Sincronizzazione non riuscita: il bot non ha potuto leggere gli abbonamenti.', EXPIRY_IN_PAST: 'La scadenza è nel passato.', @@ -158,7 +193,7 @@ const copy = { intro: 'Instance-owner area. Actions here affect every server connected to the bot.', dashboard: 'Dashboard', pricing: 'Pricing', - tabs: { servers: 'Servers & plans', contacts: 'Contacts', discord: 'Discord subscriptions', status: 'Notifications', blacklist: 'Blacklist', audit: 'Audit' }, + tabs: { servers: 'Servers & plans', monitor: 'Monitoring', logs: 'Logs', contacts: 'Contacts', discord: 'Discord subscriptions', status: 'Notifications', blacklist: 'Blacklist', audit: 'Audit' }, liveServers: 'Connected servers', liveServersText: 'Live list from the bot Discord session. Assign each server’s plan from here.', members: 'members', @@ -193,6 +228,38 @@ const copy = { url: 'Link: Discord server, website or form (https)', save: 'Save contacts' }, + monitor: { + title: 'Status and alarms', + text: 'The API checks the bot every 30 seconds. If it does not answer for about two minutes, an alarm is posted to the webhook below, and another one when it is back. Alarms use a webhook because they must arrive precisely when the bot is not running.', + bot: 'Bot', + up: 'Connected', + down: 'Unreachable', + unknown: 'Checking', + since: 'since', + lastCheck: 'last check', + servers: 'servers', + webhook: 'Discord webhook for alarms', + webhookHelp: 'In a private channel: Edit channel → Integrations → Webhooks → New webhook → Copy URL.', + webhookSet: 'Webhook configured', + webhookNotSet: 'No webhook: alarms are only kept in the event log.', + save: 'Save webhook', + remove: 'Remove', + test: 'Send test', + testOk: 'Test message sent.', + events: 'Instance events', + eventsText: 'Bot startup and errors, servers added, removed or rejected, quotas reached, bot down and restored. Kept for 90 days.', + noEvents: 'No events recorded.', + refresh: 'Refresh' + }, + logs: { + title: 'Recent logs', + text: 'The latest log lines of the bot and the API since their last restart, with tokens and secrets redacted. They stay in memory only; Docker log files are capped at 50 MB per service.', + service: 'Service', + level: 'Minimum level', + empty: 'No lines for this filter.', + refresh: 'Refresh', + auto: 'Auto refresh' + }, discord: { title: 'Discord subscriptions', text: 'Paid plans are sold only as per-server subscriptions through Discord Premium Apps, at the same prices as the pricing page. Create the SKUs in the Developer Portal (Monetization) and paste their IDs here: server plans update by themselves every two minutes.', @@ -248,6 +315,9 @@ const copy = { guild: 'Server', errors: { INVALID_SKU: 'Invalid SKU ID (17-20 digits).', + INVALID_WEBHOOK: 'Invalid webhook URL: copy it from Integrations → Webhooks.', + WEBHOOK_NOT_SET: 'No webhook configured.', + WEBHOOK_FAILED: 'Discord rejected the message: check that the webhook still exists.', DUPLICATE_SKU: 'The same SKU cannot be used for two plans.', DISCORD_SYNC_FAILED: 'Sync failed: the bot could not read the subscriptions.', EXPIRY_IN_PAST: 'The expiry date is in the past.', @@ -315,6 +385,8 @@ export default function SuperConsole({ locale }: { locale: Locale }) { const tabs: Array<{ key: Tab; icon: IconName; label: string; badge?: number }> = [ { key: 'servers', icon: 'grid', label: c.tabs.servers }, + { key: 'monitor', icon: 'globe', label: c.tabs.monitor }, + { key: 'logs', icon: 'code', label: c.tabs.logs }, { key: 'contacts', icon: 'users', label: c.tabs.contacts }, { key: 'discord', icon: 'key', label: c.tabs.discord }, { key: 'status', icon: 'bolt', label: c.tabs.status }, @@ -362,6 +434,8 @@ export default function SuperConsole({ locale }: { locale: Locale }) { {tab === 'servers' && } + {tab === 'monitor' && } + {tab === 'logs' && } {tab === 'contacts' && } {tab === 'discord' && } {tab === 'status' && } @@ -498,6 +572,129 @@ function PlanEditor({ guild, c, locale, action, onDone }: { guild: Guild; c: Cop ); } +/* ------------------------------------------------------------------ monitor */ + +type MonitorEvent = { id: string; source: string; level: string; title: string; description: string | null; fields: Array<{ name: string; value: string }>; createdAt: string }; +type Monitor = { + bot: { status: 'unknown' | 'up' | 'down'; since: string; lastCheck: string | null; guilds: number | null }; + events: MonitorEvent[]; + webhook: { configured: boolean; hint: string | null }; +}; + +function MonitorTab({ c, action, fmt, setStatus }: { c: Copy; action: Action; fmt: (value: string) => string; setStatus: SetStatus }) { + const [monitor, setMonitor] = useState(null); + const [url, setUrl] = useState(''); + const load = useCallback(() => fetch('/backend/api/super/monitor').then((response) => response.ok ? response.json() : null).then((value) => { if (value) setMonitor(value); }).catch(() => null), []); + + useEffect(() => { + void load(); + const timer = window.setInterval(() => void load(), 30_000); + return () => window.clearInterval(timer); + }, [load]); + + const saveWebhook = (value: string | null) => action('/backend/api/super/alert-webhook', { method: 'PUT', body: JSON.stringify({ url: value }) }, { reload: false }) + .then(() => { setUrl(''); void load(); }, ignore); + const test = () => action('/backend/api/super/alert-webhook/test', { method: 'POST', body: '{}' }, { reload: false, quiet: true }) + .then(() => setStatus({ kind: 'ok', text: c.monitor.testOk }), ignore); + + const bot = monitor?.bot; + const label = bot ? { up: c.monitor.up, down: c.monitor.down, unknown: c.monitor.unknown }[bot.status] : c.monitor.unknown; + + return ( + <> +
+
+
{c.tabs.monitor}

{c.monitor.title}

{c.monitor.text}

+ +
+
+
+ + {!monitor?.webhook.configured &&

{c.monitor.webhookNotSet}

} +
+ + {monitor?.webhook.configured && } + {monitor?.webhook.configured && } +
+
+ +
+
{c.tabs.monitor}

{c.monitor.events}

{c.monitor.eventsText}

+ {monitor && !monitor.events.length &&
{c.monitor.noEvents}
} +
+ {monitor?.events.map((event) => ( +
+ {event.level} +
+ {event.title} + {event.description && {event.description}} + {event.fields.map((field) => {field.name}: {field.value})} +
+ {event.source} + +
+ ))} +
+
+ + ); +} + +/* --------------------------------------------------------------------- logs */ + +type LogLine = { time: string; level: string; msg: string; context: Record | null }; + +function LogsTab({ c, fmt }: { c: Copy; fmt: (value: string) => string }) { + const [service, setService] = useState<'bot' | 'api'>('bot'); + const [level, setLevel] = useState('info'); + const [auto, setAuto] = useState(false); + const [lines, setLines] = useState(null); + const [failed, setFailed] = useState(false); + + const load = useCallback(() => { + const query = new URLSearchParams({ service, level, limit: '300' }); + return fetch(`/backend/api/super/logs?${query}`) + .then((response) => response.ok ? response.json() : Promise.reject(new Error(String(response.status)))) + .then((value: { entries: LogLine[] }) => { setLines(value.entries); setFailed(false); }) + .catch(() => setFailed(true)); + }, [service, level]); + + useEffect(() => { void load(); }, [load]); + useEffect(() => { + if (!auto) return; + const timer = window.setInterval(() => void load(), 10_000); + return () => window.clearInterval(timer); + }, [auto, load]); + + return ( +
+
{c.tabs.logs}

{c.logs.title}

{c.logs.text}

+
+ + + + +
+ {failed &&
{c.errors.BOT_UNAVAILABLE}
} + {lines && !lines.length &&
{c.logs.empty}
} +
+ {lines?.map((line, index) => ( +
+ {line.level}{line.msg || '—'} + {line.context && 
{JSON.stringify(line.context, null, 2)}
} +
+ ))} +
+
+ ); +} + /* ----------------------------------------------------------------- contacts */ function ContactsTab({ data, c, action }: Pick) { diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 9378aee..c9cdc61 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -685,6 +685,24 @@ textarea { min-height: 84px; resize: vertical; line-height: 1.5; } .segmented button span { color: var(--faint); font-size: .72rem; } .sku-form { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 0 12px; align-items: end; margin-bottom: 18px; padding: 4px 16px 16px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--bg-raised); } .sku-form > .button { margin: 14px 0; } +.monitor-status { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin-bottom: 18px; padding: 14px 16px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--bg-raised); color: var(--text-2); font-size: .88rem; } +.monitor-status strong { color: var(--text); } +.status-dot { width: 10px; height: 10px; border-radius: 50%; background: var(--faint); } +.status-up { background: #3ccf8e; box-shadow: 0 0 0 4px rgba(60,207,142,.15); } +.status-down { background: #f2555a; box-shadow: 0 0 0 4px rgba(242,85,90,.15); } +.level-badge { display: inline-block; min-width: 52px; padding: 2px 8px; border-radius: 999px; background: var(--surface-3); color: var(--text-2); font: 600 .68rem var(--font-mono-stack); text-align: center; text-transform: uppercase; } +.level-ok { background: rgba(60,207,142,.15); color: #3ccf8e; } +.level-warn { background: rgba(245,165,36,.15); color: #f5a524; } +.level-error, .level-fatal { background: rgba(242,85,90,.15); color: #ff9da0; } +.event-row { grid-template-columns: auto minmax(0, 1fr) auto auto !important; align-items: start; } +.event-row > div { min-width: 0; display: flex; flex-direction: column; gap: 3px; } +.event-row > div span { color: var(--muted); font-size: .8rem; overflow-wrap: anywhere; } +.log-lines { display: flex; flex-direction: column; border: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; } +.log-line + .log-line { border-top: 1px solid var(--line); } +.log-line summary { display: grid; grid-template-columns: auto auto minmax(0, 1fr); gap: 12px; align-items: center; padding: 8px 12px; cursor: pointer; font-size: .82rem; } +.log-line .log-msg { overflow: hidden; color: var(--text); text-overflow: ellipsis; white-space: nowrap; } +.log-line pre { margin: 0; padding: 10px 14px; overflow-x: auto; background: var(--bg-raised); color: var(--text-2); font: .74rem/1.5 var(--font-mono-stack); } +@media (max-width: 640px) { .log-line summary { grid-template-columns: auto minmax(0, 1fr); } .log-line summary time { grid-column: 1 / -1; } .event-row { grid-template-columns: auto minmax(0, 1fr) !important; } } /* Guild plan & branding */ .guild-card-tags { display: flex; flex-wrap: wrap; gap: 6px; } diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index b2aa326..9896d00 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -1,7 +1,15 @@ name: sentinel +# Docker keeps container logs in local files: cap them so they never fill the disk. +x-logging: &logging + driver: json-file + options: + max-size: "10m" + max-file: "5" + services: postgres: + logging: *logging image: postgres:17-alpine restart: unless-stopped environment: @@ -25,6 +33,7 @@ services: - /var/run/postgresql:rw,nosuid,size=8m redis: + logging: *logging image: redis:8-alpine restart: unless-stopped secrets: [redis_password] @@ -52,6 +61,7 @@ services: - /tmp:rw,noexec,nosuid,size=16m migrate: + logging: *logging image: ${SENTINEL_MIGRATE_IMAGE:?SENTINEL_MIGRATE_IMAGE is required} restart: "no" secrets: [postgres_admin_password] @@ -76,6 +86,7 @@ services: cpus: 1.0 secure-db: + logging: *logging image: postgres:17-alpine restart: "no" environment: @@ -100,6 +111,7 @@ services: pids_limit: 50 api: + logging: *logging image: ${SENTINEL_APP_IMAGE:?SENTINEL_APP_IMAGE is required} restart: unless-stopped secrets: [discord_client_secret, session_secret, log_data_encryption_key, bot_internal_api_key, api_db_password] @@ -144,6 +156,7 @@ services: cpus: 1.0 bot: + logging: *logging image: ${SENTINEL_APP_IMAGE:?SENTINEL_APP_IMAGE is required} restart: unless-stopped secrets: [discord_token, bot_internal_api_key, log_data_encryption_key, bot_db_password, redis_password] @@ -184,6 +197,7 @@ services: cpus: 1.5 web: + logging: *logging image: ${SENTINEL_APP_IMAGE:?SENTINEL_APP_IMAGE is required} restart: unless-stopped environment: @@ -213,6 +227,7 @@ services: cpus: 1.0 edge: + logging: *logging image: nginx:1.29-alpine restart: unless-stopped user: "101:101" diff --git a/deploy/runtime/harden-users.sh b/deploy/runtime/harden-users.sh index dfb3944..ce19dae 100755 --- a/deploy/runtime/harden-users.sh +++ b/deploy/runtime/harden-users.sh @@ -22,13 +22,15 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "GuildSettings", "LogRoute", "LogEvent", "MessageSnapshot", "PanelAudit", "PanelSession", "PanelRoleBinding", "InstallBlock", "SuperAdminAudit", - "InstanceConfig", "StatusChannel", "WaitlistEntry", "InstallGrant", "Coupon" + "InstanceConfig", "StatusChannel", "WaitlistEntry", "InstallGrant", "Coupon", + "InstanceEvent" TO sentinel_api; GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "GuildSettings", "LogRoute", "LogEvent", "MessageSnapshot" TO sentinel_bot; GRANT SELECT ON TABLE "InstallBlock", "InstallGrant", "StatusChannel" TO sentinel_bot; +GRANT INSERT ON TABLE "InstanceEvent" TO sentinel_bot; REVOKE ALL ON TABLE "PanelAudit", "PanelSession", "PanelRoleBinding", "SuperAdminAudit", "InstanceConfig", "WaitlistEntry", "Coupon" FROM sentinel_bot; ALTER ROLE sentinel_api NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION; diff --git a/docker-compose.yml b/docker-compose.yml index 51ad172..657dc84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,13 @@ +# Docker keeps container logs in local files: cap them so they never fill the disk. +x-logging: &logging + driver: json-file + options: + max-size: "10m" + max-file: "5" + services: postgres: + logging: *logging image: postgres:17-alpine restart: unless-stopped environment: @@ -23,6 +31,7 @@ services: - /var/run/postgresql:rw,nosuid,size=8m redis: + logging: *logging image: redis:8-alpine restart: unless-stopped secrets: [redis_password] @@ -50,6 +59,7 @@ services: - /tmp:rw,noexec,nosuid,size=16m migrate: + logging: *logging build: context: . target: migrate @@ -74,6 +84,7 @@ services: pids_limit: 100 secure-db: + logging: *logging image: postgres:17-alpine restart: "no" environment: @@ -98,6 +109,7 @@ services: pids_limit: 50 api: + logging: *logging build: context: . target: runtime @@ -136,6 +148,7 @@ services: cpus: 1.0 bot: + logging: *logging build: context: . target: runtime @@ -172,6 +185,7 @@ services: cpus: 1.5 web: + logging: *logging build: context: . target: runtime @@ -197,6 +211,7 @@ services: cpus: 1.0 caddy: + logging: *logging profiles: ["standalone"] image: caddy:2.10-alpine restart: unless-stopped diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 805f138..03b4278 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -278,6 +278,13 @@ pg_restore --clean --if-exists -d "$DATABASE_URL" restore.dump shred -u restore.dump ``` +## Monitoring + +- **Super console → Monitoring:** bot status, *down*/*restored* alarms to a Discord webhook (set it there) and the instance event feed. +- **Super console → Logs:** latest bot and API log lines, redacted, in memory only. +- **Docker logs** are capped by the Compose file (`json-file`, 5 × 10 MB per service). After updating `deploy/docker-compose.prod.yml`, reinstall it on the VPS and recreate the containers. +- **External monitor:** check `https:///backend/health` (API and database) and `https:///backend/health/bot` (bot connected to Discord) from a service outside the VPS. + ## 10. Repository protection Configured on GitHub for `haxurus/Sentinel`: diff --git a/packages/db/prisma/migrations/20261020120000_instance_monitoring/migration.sql b/packages/db/prisma/migrations/20261020120000_instance_monitoring/migration.sql new file mode 100644 index 0000000..5a9c58b --- /dev/null +++ b/packages/db/prisma/migrations/20261020120000_instance_monitoring/migration.sql @@ -0,0 +1,30 @@ +-- Super console monitoring: instance events and the alarm webhook. +ALTER TABLE "InstanceConfig" ADD COLUMN "alertWebhookUrl" TEXT; + +CREATE TABLE "InstanceEvent" ( + "id" TEXT NOT NULL, + "source" TEXT NOT NULL, + "level" TEXT NOT NULL, + "title" TEXT NOT NULL, + "details" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "InstanceEvent_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX "InstanceEvent_createdAt_idx" ON "InstanceEvent"("createdAt" DESC); + +-- Production roles already exist when this runs; fresh installs get the same +-- grants from harden-users.sh right after the migrations. The bot may only +-- insert (createMany, no RETURNING): it never reads the events back. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'sentinel_api') THEN + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "InstanceEvent" TO sentinel_api; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'sentinel_bot') THEN + REVOKE ALL ON TABLE "InstanceEvent" FROM sentinel_bot; + GRANT INSERT ON TABLE "InstanceEvent" TO sentinel_bot; + END IF; +END +$$; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 02c19f4..900ae65 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -212,6 +212,8 @@ model InstanceConfig { // Discord Premium Apps SKU IDs by plan and billing period: // { "TIER1": { "MONTHLY": "", "YEARLY": "" }, ... } discordSkus Json @default("{}") + // Discord webhook for alarms that must work when the bot is down (encrypted). + alertWebhookUrl String? updatedAt DateTime @updatedAt } @@ -274,3 +276,17 @@ model Coupon { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt } + +// Instance-wide notices shown in the super console: bot startup and errors, +// servers added, removed or rejected, quotas, bot down/restored. The bot can +// only insert; description and fields are encrypted in `details`. +model InstanceEvent { + id String @id @default(cuid()) + source String + level String + title String + details Json + createdAt DateTime @default(now()) + + @@index([createdAt(sort: Desc)]) +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 9f79146..c84f2c5 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -82,3 +82,4 @@ export const EVENT_CATALOG: EventDefinition[] = [ export const eventDefinition = (key: string) => EVENT_CATALOG.find((event) => event.key === key); export * from './plans.js'; +export * from './log-buffer.js'; diff --git a/packages/shared/src/log-buffer.ts b/packages/shared/src/log-buffer.ts new file mode 100644 index 0000000..abca4fe --- /dev/null +++ b/packages/shared/src/log-buffer.ts @@ -0,0 +1,63 @@ +/** + * The latest log lines of a process, kept in memory for the super console. + * It is a pino destination: pass it to pino.multistream next to stdout. Lines + * are redacted before they are stored and nothing is written to disk. + */ + +export type LogLevelName = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal'; +export type LogEntry = { time: string; level: LogLevelName; msg: string; context: Record | null }; + +const LEVELS: Record = { 10: 'trace', 20: 'debug', 30: 'info', 40: 'warn', 50: 'error', 60: 'fatal' }; +export const LOG_LEVEL_RANK: Record = { trace: 10, debug: 20, info: 30, warn: 40, error: 50, fatal: 60 }; +const DROPPED_KEYS = new Set(['time', 'level', 'msg', 'pid', 'hostname', 'v']); +const MAX_LINE = 4_000; + +export function createLogBuffer(capacity = 2_000, redact: (value: string) => string = (value) => value) { + const entries: LogEntry[] = []; + + const push = (line: string) => { + let parsed: Record; + try { + parsed = JSON.parse(line) as Record; + } catch { + parsed = { level: 30, msg: line }; + } + const context: Record = {}; + for (const [key, value] of Object.entries(parsed)) if (!DROPPED_KEYS.has(key)) context[key] = value; + const rawContext = Object.keys(context).length ? redact(JSON.stringify(context)).slice(0, MAX_LINE) : null; + let safeContext: Record | null = null; + if (rawContext) { + try { + safeContext = JSON.parse(rawContext) as Record; + } catch { + safeContext = { truncated: rawContext }; + } + } + entries.push({ + time: new Date(typeof parsed.time === 'number' ? parsed.time : Date.now()).toISOString(), + level: LEVELS[Number(parsed.level)] ?? 'info', + msg: redact(String(parsed.msg ?? '')).slice(0, MAX_LINE), + context: safeContext + }); + if (entries.length > capacity) entries.splice(0, entries.length - capacity); + }; + + return { + /** pino destination interface. */ + write(chunk: string) { + for (const line of String(chunk).split('\n')) if (line.trim()) push(line); + }, + /** Newest first, at or above `minLevel`. */ + list(minLevel: LogLevelName = 'info', limit = 500) { + const rank = LOG_LEVEL_RANK[minLevel]; + const out: LogEntry[] = []; + for (let index = entries.length - 1; index >= 0 && out.length < limit; index -= 1) { + const entry = entries[index]!; + if (LOG_LEVEL_RANK[entry.level] >= rank) out.push(entry); + } + return out; + } + }; +} + +export type LogBuffer = ReturnType;