From 22f7b54fcc204760ac2dbd4157fa3a9b6f7acb25 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 00:48:41 +0100 Subject: [PATCH 01/21] fix(live-alerts): isolate a rejected Twitch login or Kick slug from the shared poll One login or slug that the platform answers with a 4xx used to fail the whole batch, so no server got go-live posts. A rejected batch is now bisected down to the offending value, which is quarantined with a growing backoff (10 min up to 6 h) and logged with the servers that saved it. The other values keep polling. 5xx, 401 and 429 still fail the poll, and so does a poll where many different values are rejected (a request-level problem such as bad app credentials). --- src/modules/kickAlerts.js | 37 +++-- src/modules/lib/batchIsolation.js | 97 +++++++++++++ src/modules/twitchAlerts.js | 54 +++++--- test/liveAlertsIsolation.test.js | 223 ++++++++++++++++++++++++++++++ 4 files changed, 385 insertions(+), 26 deletions(-) create mode 100644 src/modules/lib/batchIsolation.js create mode 100644 test/liveAlertsIsolation.test.js diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index 45913308..4ea0f540 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -17,6 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; +import { createQuarantine, fetchIsolated } from './lib/batchIsolation.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -97,20 +98,26 @@ async function getChannels(slugs) { token = { value: null, expiresAt: 0 }; // force a refresh next call throw new Error('Kick 401'); } - if (!res.ok) throw new Error(`Kick /channels ${res.status}`); + if (!res.ok) throw Object.assign(new Error(`Kick /channels ${res.status}`), { status: res.status }); return (await res.json()).data ?? []; } +/** + * Slugs the Kick API rejected (a 4xx for a request holding only that slug), left out of the + * shared poll with a growing backoff so one saved value cannot stop alerts for every server. + */ +export const rejectedSlugs = createQuarantine(); + /** * Live channels for a set of slugs, keyed by lowercased slug. Only currently-live - * channels are included. + * channels are included. A slug that Kick rejects is isolated (see fetchIsolated), skipped + * and reported through `onRejected(slug, err, waitMs)`. */ -export async function fetchLiveChannels(slugs) { +export async function fetchLiveChannels(slugs, { onRejected } = {}) { const out = new Map(); - for (let i = 0; i < slugs.length; i += 50) { - for (const ch of await getChannels(slugs.slice(i, i + 50))) { - if (ch?.stream?.is_live && ch.slug) out.set(String(ch.slug).toLowerCase(), ch); - } + const found = await fetchIsolated(slugs, 50, getChannels, { quarantine: rejectedSlugs, onRejected }); + for (const ch of found) { + if (ch?.stream?.is_live && ch.slug) out.set(String(ch.slug).toLowerCase(), ch); } return out; } @@ -166,7 +173,8 @@ export function buildPayload(channel, alert) { // --- poll loop ------------------------------------------------------- -async function tick() { +/** One poll; exported for tests. */ +export async function tick() { if (!allClientsReady() || !config.kickEnabled) return; const jobs = []; @@ -178,15 +186,26 @@ async function tick() { if (!jobs.length) return; const slugs = [...new Set(jobs.map((j) => j.alert.slug))]; + rejectedSlugs.prune(new Set(slugs)); let live; try { - live = await fetchLiveChannels(slugs); + live = await fetchLiveChannels(slugs, { + onRejected: (slug, err, waitMs) => { + const guilds = [...new Set(jobs.filter((j) => j.alert.slug === slug).map((j) => j.guildId))]; + log.warn( + 'kick-alerts', + `Kick rejected slug "${slug}" (${err.message}); skipping it for ${Math.round(waitMs / 60_000)} min. Saved in: ${guilds.join(', ')}` + ); + }, + }); } catch (err) { log.error('kick-alerts', 'poll failed:', err.message); return; } for (const { guildId, alert } of jobs) { + // Rejected by Kick: neither live nor offline as far as we know, so leave its state alone. + if (rejectedSlugs.has(alert.slug)) continue; const channel = live.get(alert.slug); if (!channel) { const row = await seenRow(guildId, SCOPE, alert.slug); diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js new file mode 100644 index 00000000..9cb8ff60 --- /dev/null +++ b/src/modules/lib/batchIsolation.js @@ -0,0 +1,97 @@ +// Isolate one rejected item from a shared, batched platform request. +// +// The Twitch and Kick alert modules poll every server's streamers together. If the platform +// answers a batch with a 4xx because of one stored login or slug, that single value must not stop +// the poll for every other server. fetchIsolated() splits such a batch in half until the rejected +// item stands alone, quarantines that item with a growing backoff, and keeps the results of the +// rest. Other failures (5xx, network errors, 401, 429) still throw: they are not about one item +// and retrying the whole poll later is right. + +/** More rejected items than this in one poll means the request is at fault, not the items. */ +const MAX_REJECTED = 10; + +/** Status codes that mean "the platform rejected the request itself", not "try again later". */ +export function isRejectedStatus(status) { + return Number.isInteger(status) && status >= 400 && status < 500 && status !== 401 && status !== 429; +} + +/** + * A set of items to leave out of polls for a while. Each repeated rejection doubles the wait. + * @param {{ baseMs?: number, maxMs?: number, now?: () => number }} [o] + */ +export function createQuarantine({ baseMs = 10 * 60_000, maxMs = 6 * 60 * 60_000, now = Date.now } = {}) { + /** @type {Map} */ + const entries = new Map(); + return { + /** True while the item is waiting out its backoff. */ + has(key) { + const e = entries.get(key); + return Boolean(e && e.until > now()); + }, + /** Quarantine the item (again); returns the wait in ms. */ + add(key) { + const strikes = (entries.get(key)?.strikes ?? 0) + 1; + const wait = Math.min(maxMs, baseMs * 2 ** (strikes - 1)); + entries.set(key, { until: now() + wait, strikes }); + return wait; + }, + /** Forget an item that the platform accepted again. */ + clear(key) { + entries.delete(key); + }, + /** Drop entries that are over and no longer stored anywhere (keeps the map small). */ + prune(keep) { + for (const [key, e] of entries) if (e.until <= now() && !keep.has(key)) entries.delete(key); + }, + get size() { + return entries.size; + }, + }; +} + +/** + * Run `fetchBatch` over `items` in chunks of `size`, bisecting any chunk the platform rejects. + * + * @template T + * @param {string[]} items + * @param {number} size + * @param {(batch: string[]) => Promise} fetchBatch throws an error with `.status` on a non-OK answer + * @param {{ quarantine: ReturnType, onRejected?: (item: string, err: Error, waitMs: number) => void, maxRejected?: number }} o + * @returns {Promise} the results of every accepted item + */ +export async function fetchIsolated( + items, + size, + fetchBatch, + { quarantine, onRejected, maxRejected = MAX_REJECTED } +) { + const out = []; + const added = []; + const run = async (batch) => { + try { + out.push(...(await fetchBatch(batch))); + } catch (err) { + if (!isRejectedStatus(err?.status)) throw err; + if (batch.length === 1) { + // Many different items rejected in one poll: the problem is the request (a bad app + // credential, say), not the items. Undo and fail the poll like any other error. + if (added.length >= maxRejected) { + for (const item of added) quarantine.clear(item); + throw err; + } + added.push(batch[0]); + const waitMs = quarantine.add(batch[0]); + onRejected?.(batch[0], err, waitMs); + return; + } + const mid = Math.ceil(batch.length / 2); + await run(batch.slice(0, mid)); + await run(batch.slice(mid)); + return; + } + for (const item of batch) quarantine.clear(item); + }; + const live = items.filter((item) => !quarantine.has(item)); + for (let i = 0; i < live.length; i += size) await run(live.slice(i, i + size)); + return out; +} diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 3abb2655..5027b4ce 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -17,6 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; +import { createQuarantine, fetchIsolated } from './lib/batchIsolation.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -92,30 +93,37 @@ async function helix(path, params) { token = { value: null, expiresAt: 0 }; // force refresh next call throw new Error('Twitch 401'); } - if (!res.ok) throw new Error(`Twitch ${path} ${res.status}`); + if (!res.ok) throw Object.assign(new Error(`Twitch ${path} ${res.status}`), { status: res.status }); return (await res.json()).data ?? []; } -/** Live streams for a set of logins, keyed by lowercased login. */ -export async function fetchLiveStreams(logins) { +/** + * Logins the Helix API rejected (a 4xx for a request holding only that login), left out of the + * shared poll with a growing backoff so one saved value cannot stop alerts for every server. + */ +export const rejectedLogins = createQuarantine(); + +/** + * Live streams for a set of logins, keyed by lowercased login. A login that Helix rejects is + * isolated (see fetchIsolated), skipped and reported through `onRejected(login, err, waitMs)`. + */ +export async function fetchLiveStreams(logins, { onRejected } = {}) { const out = new Map(); - for (let i = 0; i < logins.length; i += 100) { - const batch = logins.slice(i, i + 100); - for (const s of await helix('/streams', { user_login: batch })) { - out.set(s.user_login.toLowerCase(), s); - } - } + const found = await fetchIsolated(logins, 100, (batch) => helix('/streams', { user_login: batch }), { + quarantine: rejectedLogins, + onRejected, + }); + for (const s of found) out.set(s.user_login.toLowerCase(), s); return out; } -/** User records (for avatar / display name), keyed by lowercased login. */ +/** User records (for avatar / display name), keyed by lowercased login. A rejected login is just omitted. */ export async function fetchUsers(logins) { const out = new Map(); - for (let i = 0; i < logins.length; i += 100) { - for (const u of await helix('/users', { login: logins.slice(i, i + 100) })) { - out.set(u.login.toLowerCase(), u); - } - } + const found = await fetchIsolated(logins, 100, (batch) => helix('/users', { login: batch }), { + quarantine: createQuarantine(), + }); + for (const u of found) out.set(u.login.toLowerCase(), u); return out; } @@ -166,7 +174,8 @@ export function buildPayload(stream, user, alert) { // --- poll loop ------------------------------------------------------- -async function tick() { +/** One poll; exported for tests. */ +export async function tick() { if (!allClientsReady() || !config.twitchEnabled) return; // Gather every alert across enabled guilds. @@ -179,10 +188,19 @@ async function tick() { if (!jobs.length) return; const logins = [...new Set(jobs.map((j) => j.alert.login))]; + rejectedLogins.prune(new Set(logins)); let streams; let users; try { - streams = await fetchLiveStreams(logins); + streams = await fetchLiveStreams(logins, { + onRejected: (login, err, waitMs) => { + const guilds = [...new Set(jobs.filter((j) => j.alert.login === login).map((j) => j.guildId))]; + log.warn( + 'twitch-alerts', + `Twitch rejected login "${login}" (${err.message}); skipping it for ${Math.round(waitMs / 60_000)} min. Saved in: ${guilds.join(', ')}` + ); + }, + }); users = await fetchUsers([...streams.keys()]); } catch (err) { log.error('twitch-alerts', 'poll failed:', err.message); @@ -190,6 +208,8 @@ async function tick() { } for (const { guildId, alert } of jobs) { + // Rejected by Twitch: neither live nor offline as far as we know, so leave its state alone. + if (rejectedLogins.has(alert.login)) continue; const stream = streams.get(alert.login); if (!stream) { const post = await announcedPost(guildId, alert.login); diff --git a/test/liveAlertsIsolation.test.js b/test/liveAlertsIsolation.test.js new file mode 100644 index 00000000..e19d3183 --- /dev/null +++ b/test/liveAlertsIsolation.test.js @@ -0,0 +1,223 @@ +// One rejected Twitch login or Kick slug must not stop live alerts for every other server. +process.env.TWITCH_CLIENT_ID = 'tid'; +process.env.TWITCH_CLIENT_SECRET = 'tsecret'; +process.env.KICK_CLIENT_ID = 'kid'; +process.env.KICK_CLIENT_SECRET = 'ksecret'; +await import('./helpers/tmpDb.js'); +const { default: test } = await import('node:test'); +const { default: assert } = await import('node:assert/strict'); +const { runtime } = await import('../src/runtime.js'); +const { setGuildModule } = await import('../src/db/modules.js'); +const { createQuarantine, fetchIsolated, isRejectedStatus } = + await import('../src/modules/lib/batchIsolation.js'); +const twitch = await import('../src/modules/twitchAlerts.js'); +const kick = await import('../src/modules/kickAlerts.js'); + +const CH_A = '100000000000000001'; +const ATTACKER = '900000000000000501'; +const VICTIM = '900000000000000502'; + +const rejected = (status) => Object.assign(new Error(`rejected ${status}`), { status }); + +test('isRejectedStatus: 4xx except 401/429 only', () => { + assert.equal(isRejectedStatus(400), true); + assert.equal(isRejectedStatus(404), true); + assert.equal(isRejectedStatus(401), false); + assert.equal(isRejectedStatus(429), false); + assert.equal(isRejectedStatus(500), false); + assert.equal(isRejectedStatus(undefined), false); +}); + +test('fetchIsolated: bisects to the rejected item and keeps the rest', async () => { + const quarantine = createQuarantine(); + const calls = []; + const items = Array.from({ length: 8 }, (_, i) => `item${i}`); + const got = await fetchIsolated( + items, + 8, + async (batch) => { + calls.push(batch.length); + if (batch.includes('item5')) throw rejected(400); + return batch; + }, + { quarantine } + ); + assert.deepEqual(got.sort(), items.filter((i) => i !== 'item5').sort()); + assert.equal(quarantine.has('item5'), true); + assert.equal(quarantine.has('item4'), false); + assert.ok(calls.length <= 1 + 2 * 3 + 1, `bounded request count, got ${calls.length}`); +}); + +test('fetchIsolated: a non-4xx failure still fails the whole poll', async () => { + const quarantine = createQuarantine(); + await assert.rejects( + fetchIsolated(['a', 'b'], 2, async () => Promise.reject(rejected(503)), { quarantine }), + /rejected 503/ + ); + assert.equal(quarantine.size, 0); + await assert.rejects( + fetchIsolated(['a', 'b'], 2, async () => Promise.reject(rejected(429)), { quarantine }), + /rejected 429/ + ); + assert.equal(quarantine.size, 0); +}); + +test('fetchIsolated: when everything is rejected the request is at fault, nothing is quarantined', async () => { + const quarantine = createQuarantine(); + const items = Array.from({ length: 40 }, (_, i) => `item${i}`); + let calls = 0; + await assert.rejects( + fetchIsolated( + items, + 40, + async () => { + calls++; + throw rejected(403); + }, + { quarantine } + ), + /rejected 403/ + ); + assert.equal(quarantine.size, 0); + assert.ok(calls < 40, `gave up early, made ${calls} calls`); +}); + +test('quarantine: backoff doubles, expires, and clears', () => { + let now = 1_000; + const q = createQuarantine({ baseMs: 100, maxMs: 350, now: () => now }); + assert.equal(q.add('x'), 100); + assert.equal(q.has('x'), true); + now += 101; + assert.equal(q.has('x'), false); + assert.equal(q.add('x'), 200); + assert.equal(q.add('x'), 350); // capped + q.clear('x'); + assert.equal(q.has('x'), false); + assert.equal(q.add('x'), 100); +}); + +// --- tick-level: the victim still gets an alert while a poisoned record is stored ------------ + +function fakeGuild(id, sent) { + const channel = { + isTextBased: () => true, + permissionsFor: () => ({ has: () => true }), + send: async (payload) => { + sent.push({ guild: id, payload }); + return { id: `msg-${sent.length}` }; + }, + }; + return { + id, + channels: { cache: new Map([[CH_A, channel]]), fetch: async () => channel }, + members: { me: {} }, + }; +} + +function installClient(sent) { + const guilds = new Map([ + [ATTACKER, fakeGuild(ATTACKER, sent)], + [VICTIM, fakeGuild(VICTIM, sent)], + ]); + runtime.client = { isReady: () => true, guilds: { cache: guilds } }; + for (const g of guilds.values()) g.client = runtime.client; +} + +test('twitch tick: a rejected login does not stop alerts for another server', async () => { + const sent = []; + installClient(sent); + await setGuildModule(ATTACKER, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: '_poison', channelId: CH_A }] }, + }); + await setGuildModule(VICTIM, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: 'victim1', channelId: CH_A }] }, + }); + + const realFetch = globalThis.fetch; + const streamUrls = []; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.twitch.tv') return Response.json({ access_token: 't', expires_in: 3600 }); + const logins = u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login'); + if (logins.some((l) => l.startsWith('_'))) return new Response('{}', { status: 400 }); + if (u.pathname.endsWith('/streams')) { + streamUrls.push(logins); + return Response.json({ + data: logins + .filter((l) => l === 'victim1') + .map((l) => ({ + id: 's1', + user_login: l, + user_name: l, + title: 'hi', + game_name: 'g', + viewer_count: 3, + started_at: '2026-11-16T00:00:00Z', + thumbnail_url: '', + })), + }); + } + return Response.json({ data: [] }); + }; + try { + await twitch.tick(); + assert.equal(sent.length, 1, 'victim announced on the first poll'); + assert.equal(sent[0].guild, VICTIM); + assert.equal(twitch.rejectedLogins.has('_poison'), true); + + // Next poll: the poisoned login is skipped, no more requests for it. + streamUrls.length = 0; + await twitch.tick(); + assert.ok(streamUrls.every((b) => !b.includes('_poison'))); + assert.equal(sent.length, 1, 'no duplicate announcement'); + } finally { + globalThis.fetch = realFetch; + } +}); + +test('kick tick: a rejected slug does not stop alerts for another server', async () => { + const sent = []; + installClient(sent); + await setGuildModule(ATTACKER, 'kick-alerts', { + enabled: true, + config: { alerts: [{ slug: '-poison', channelId: CH_A }] }, + }); + await setGuildModule(VICTIM, 'kick-alerts', { + enabled: true, + config: { alerts: [{ slug: 'victimk', channelId: CH_A }] }, + }); + + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.kick.com') return Response.json({ access_token: 't', expires_in: 3600 }); + const slugs = u.searchParams.getAll('slug'); + if (slugs.some((s) => s.startsWith('-'))) return new Response('{}', { status: 400 }); + return Response.json({ + data: slugs + .filter((s) => s === 'victimk') + .map((slug) => ({ + slug, + stream_title: 'hi', + category: { name: 'g' }, + stream: { + is_live: true, + start_time: '2026-11-16T00:00:00Z', + viewer_count: 2, + url: 'https://kick.com/victimk', + }, + })), + }); + }; + try { + await kick.tick(); + assert.equal(sent.length, 1, 'victim announced on the first poll'); + assert.equal(sent[0].guild, VICTIM); + assert.equal(kick.rejectedSlugs.has('-poison'), true); + } finally { + globalThis.fetch = realFetch; + runtime.client = null; + } +}); From 43c3cda2ac00ce0e7a8a3af04bb781d4191428dc Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 00:51:59 +0100 Subject: [PATCH 02/21] fix(tickets): bound the member lookups one DM can cause A DM used to fetch the sender as a member in every ticket-enabled server of the instance. The lookup now uses the member cache first (a server whose member list is fully cached answers with no request), caps REST fetches at 8 per DM and 120 per minute for the bot, remembers "not a member" for two minutes, tries servers where the user was found before first, and reuses a sender's answer for 10 seconds. Without the Server Members intent senders are still found by the capped fetches; a message that could not check every server yet asks the sender to try again, which continues with the unchecked servers. --- docs/modules/tickets.md | 8 + src/bot/events/dmTickets.js | 16 +- src/modules/tickets.js | 154 ++++++++++++++-- test/ticketGuildLookup.noIntent.test.js | 71 ++++++++ test/ticketGuildLookup.test.js | 226 ++++++++++++++++++++++++ 5 files changed, 458 insertions(+), 17 deletions(-) create mode 100644 test/ticketGuildLookup.noIntent.test.js create mode 100644 test/ticketGuildLookup.test.js diff --git a/docs/modules/tickets.md b/docs/modules/tickets.md index c0831a48..e0df6f95 100644 --- a/docs/modules/tickets.md +++ b/docs/modules/tickets.md @@ -33,5 +33,13 @@ server. Staff reply as **Staff** from the ticket page; the member sees Sylo's DM **Close ticket** sends a final reply (or the default closing notice if the box is empty) and closes it. The member can open a new one by DMing again. +To find the servers a member shares with the bot, Sylo uses its member cache first. A server +whose full member list is cached (Server Members intent on) answers without any request to +Discord. For the others it asks Discord for the one member, at most 8 servers per message, at most +120 requests a minute for the whole bot, and it remembers "not a member" for two minutes. Without +the Server Members intent a member is still found this way; if a message could not check every +server yet, the member is asked to send it again a few seconds later, which continues with the +servers not checked yet. + `/forget` removes the member's ticket history: every message in their tickets, staff replies included. Replies already delivered by DM are not clawed back. diff --git a/src/bot/events/dmTickets.js b/src/bot/events/dmTickets.js index 1354a72c..a9107a80 100644 --- a/src/bot/events/dmTickets.js +++ b/src/bot/events/dmTickets.js @@ -5,7 +5,7 @@ import { guildKey, guildByKey, platformOfClient } from '../../platform/platformOf.js'; import { config } from '../../config.js'; import { Events, StringSelectMenuBuilder, ActionRowBuilder } from 'discord.js'; -import { ticketGuildsForUser, ingestUserDM } from '../../modules/tickets.js'; +import { lookupTicketGuilds, canOpenTicketIn, ingestUserDM } from '../../modules/tickets.js'; import { getOpenTicket } from '../../db/tickets.js'; import { registerComponent } from '../lib/components.js'; import { handleQuestionnaireMessage, looksLikeQuestionnairePrompt } from '../lib/dmQuestionnaire.js'; @@ -70,10 +70,15 @@ async function handleDM(message) { if (!payload.content && payload.attachments.length === 0) return; // Only the servers on the platform this DM arrived on: a user id means nothing on the other one. - const guilds = await ticketGuildsForUser(message.author, message.client); + const { guilds, incomplete } = await lookupTicketGuilds(message.author, message.client); if (guilds.length === 0) { + // `incomplete`: the lookup is capped per message, so some servers were not checked yet. await message - .reply("I'm not set up to take messages for any server you're in right now.") + .reply( + incomplete + ? 'I could not check all servers yet. Please send your message again in a few seconds.' + : "I'm not set up to take messages for any server you're in right now." + ) .catch(() => {}); return; } @@ -113,10 +118,7 @@ async function handleDM(message) { async function handleSelect(interaction) { const picked = guildByKey(interaction.values[0]); // Still one the member may open a ticket in (a member there, tickets on): the menu could be old. - const allowed = picked - ? await ticketGuildsForUser(interaction.user, interaction.client ?? picked.client) - : []; - const guild = allowed.find((g) => guildKey(g) === guildKey(picked)); + const guild = picked && (await canOpenTicketIn(interaction.user, picked)) ? picked : null; const payload = takeStash(interaction.user.id); if (!guild || !payload) { await interaction diff --git a/src/modules/tickets.js b/src/modules/tickets.js index ec73bb88..afc268bd 100644 --- a/src/modules/tickets.js +++ b/src/modules/tickets.js @@ -1,12 +1,13 @@ // Ticket / modmail helpers. Users DM the bot; staff reply from the dashboard. // The bot relays messages both ways. Staff replies are anonymous ("Staff"). -import { allGuilds, clientOfGuild, guildByKey, guildKey } from '../platform/platformOf.js'; +import { allGuilds, clientOfGuild, guildByKey, guildKey, platformOfClient } from '../platform/platformOf.js'; import { EmbedBuilder, AttachmentBuilder } from 'discord.js'; import { config } from '../config.js'; import { idList } from '../web/lib/idList.js'; import { clampRetentionDays } from './lib/retentionDays.js'; import { isModuleEnabled, getGuildModule } from '../db/modules.js'; import { notifyTicketActivity } from '../lib/push.js'; +import { sweepMap } from '../lib/sweepMap.js'; import { openOrCreateTicket, getTicket, @@ -152,18 +153,151 @@ ${bubbles} return { filename: `ticket-${ticket.id}-transcript.html`, html }; } -/** - * Guilds where the tickets module is enabled and the user is a member. Pass the client the - * DM arrived on to look only at that platform's servers. - */ -export async function ticketGuildsForUser(user, client) { - const out = []; +// --- Which ticket servers is a DM sender in? -------------------------------------------------- +// Any account can DM the bot, so the lookup must cost little however many servers the instance +// has. Order of work: (1) members already in the cache cost nothing; (2) when the member cache of +// a server is known to be complete (GuildMembers intent on and every member cached) a miss is a +// definite "not a member"; (3) only the rest need a REST member fetch, and those are capped per +// DM, per minute for the whole bot, remembered when the member does not exist, and tried first +// where the user was found before. Without the GuildMembers intent the member caches are sparse, +// so step 3 is the only way to find a real sender: it still works, a few servers per DM, and a +// sender whose servers were not all checked is told to send the message again, which continues +// with the servers not yet checked. + +/** Most REST member fetches one DM may cause. */ +export const MEMBER_PROBES_PER_DM = 8; +/** Most REST member fetches all DMs together may cause per minute. */ +export const MEMBER_PROBES_PER_MINUTE = 120; +/** A "not a member" answer is remembered this long. */ +export const NOT_MEMBER_TTL_MS = 2 * 60_000; +/** The answer for one sender is reused this long (a per-user cooldown on the lookup). */ +export const LOOKUP_REUSE_MS = 10_000; +const HINT_TTL_MS = 24 * 60 * 60_000; + +/** `${userId}|${guildKey}` -> expiry of a "not a member" answer. @type {Map} */ +const notMember = new Map(); +/** userId -> servers the user was found in lately; only decides the order of the fetches. */ +const foundBefore = new Map(); +/** `${platform}|${userId}` -> { at, result } (result is a promise while the lookup runs). */ +const lookups = new Map(); +let probeWindow = { start: 0, count: 0 }; +sweepMap(notMember, (until, now) => until <= now); +sweepMap(foundBefore, (h, now) => now - h.at > HINT_TTL_MS); +sweepMap(lookups, (l, now) => now - l.at > LOOKUP_REUSE_MS); + +/** Test hook: forget every remembered answer and the probe budget. */ +export function _resetTicketLookup() { + notMember.clear(); + foundBefore.clear(); + lookups.clear(); + probeWindow = { start: 0, count: 0 }; +} + +const cachedMember = (guild, userId) => { + const m = guild.members.cache?.get(userId); + return m && !m.partial ? m : null; +}; + +/** Is every member of the server in the cache, so that a miss means "not a member"? */ +function memberCacheComplete(guild) { + if (platformOfClient(guild.client) !== 'discord' || !config.intentGuildMembers) return false; + return Number.isFinite(guild.memberCount) && (guild.members.cache?.size ?? 0) >= guild.memberCount; +} + +/** Take one REST member fetch from the per-minute budget; false when it is used up. */ +function takeProbe() { + const now = Date.now(); + if (now - probeWindow.start >= 60_000) probeWindow = { start: now, count: 0 }; + if (probeWindow.count >= MEMBER_PROBES_PER_MINUTE) return false; + probeWindow.count++; + return true; +} + +/** One REST member fetch. @returns {Promise} true member, false not a member, null unknown */ +async function probeMember(guild, userId) { + try { + // A platform may answer "no such member" with null instead of an error. + if (await guild.members.fetch(userId)) return true; + notMember.set(`${userId}|${guildKey(guild)}`, Date.now() + NOT_MEMBER_TTL_MS); + return false; + } catch (err) { + if (err?.code === 10007 || err?.code === 10013 || err?.status === 404) { + notMember.set(`${userId}|${guildKey(guild)}`, Date.now() + NOT_MEMBER_TTL_MS); + return false; + } + return null; // a rate limit or a network error says nothing about membership + } +} + +async function findTicketGuilds(user, client) { + const found = []; + const toProbe = []; + const now = Date.now(); for (const guild of client ? [...client.guilds.cache.values()] : allGuilds()) { + if (cachedMember(guild, user.id)) { + if (await isModuleEnabled(guildKey(guild), 'tickets')) found.push(guild); + continue; + } + if (memberCacheComplete(guild)) continue; + if ((notMember.get(`${user.id}|${guildKey(guild)}`) ?? 0) > now) continue; + toProbe.push(guild); + } + + // Servers where this user was found before go first. + const hint = foundBefore.get(user.id)?.guilds; + if (hint) toProbe.sort((a, b) => Number(hint.has(guildKey(b))) - Number(hint.has(guildKey(a)))); + + let probes = 0; + let incomplete = false; + for (const guild of toProbe) { if (!(await isModuleEnabled(guildKey(guild), 'tickets'))) continue; - const member = await guild.members.fetch(user.id).catch(() => null); - if (member) out.push(guild); + if (probes >= MEMBER_PROBES_PER_DM || !takeProbe()) { + incomplete = true; + break; + } + probes++; + const isMember = await probeMember(guild, user.id); + if (isMember) found.push(guild); + else if (isMember === null) incomplete = true; + } + + if (found.length) { + foundBefore.set(user.id, { guilds: new Set(found.map(guildKey)), at: Date.now() }); + } + return { guilds: found, incomplete }; +} + +/** + * Guilds where the tickets module is enabled and the user is a member, and whether every + * candidate server was checked (`incomplete` when the fetch caps cut the lookup short). Pass the + * client the DM arrived on to look only at that platform's servers. The answer for one user is + * reused for a few seconds, so a burst of DMs does the work once. + * @returns {Promise<{ guilds: import('discord.js').Guild[], incomplete: boolean }>} + */ +export function lookupTicketGuilds(user, client) { + const key = `${client ? platformOfClient(client) : 'all'}|${user.id}`; + const recent = lookups.get(key); + if (recent && Date.now() - recent.at < LOOKUP_REUSE_MS) return recent.result; + const result = findTicketGuilds(user, client); + lookups.set(key, { at: Date.now(), result }); + result.catch(() => lookups.delete(key)); + return result; +} + +/** + * Can this user open a ticket in this one server (tickets on, user a member)? Cheap: the member + * cache, or the DM lookup that just ran, or at most one fetch from the shared budget. + */ +export async function canOpenTicketIn(user, guild) { + if (!(await isModuleEnabled(guildKey(guild), 'tickets'))) return false; + if (cachedMember(guild, user.id)) return true; + const recent = lookups.get(`${platformOfClient(guild.client)}|${user.id}`); + if (recent && Date.now() - recent.at < LOOKUP_REUSE_MS) { + const { guilds } = await recent.result.catch(() => ({ guilds: [] })); + if (guilds.some((g) => guildKey(g) === guildKey(guild))) return true; } - return out; + if (memberCacheComplete(guild) || !takeProbe()) return false; + return (await probeMember(guild, user.id)) === true; } const ALERT_MESSAGE_MAX = 1000; diff --git a/test/ticketGuildLookup.noIntent.test.js b/test/ticketGuildLookup.noIntent.test.js new file mode 100644 index 00000000..576db11f --- /dev/null +++ b/test/ticketGuildLookup.noIntent.test.js @@ -0,0 +1,71 @@ +// Without the GuildMembers intent the member caches are sparse, so real senders are found by +// capped REST fetches instead. They must still be found. +process.env.INTENT_GUILD_MEMBERS = 'false'; +await import('./helpers/tmpDb.js'); +const { default: test } = await import('node:test'); +const { default: assert } = await import('node:assert/strict'); +const { setGuildModule } = await import('../src/db/modules.js'); +const { lookupTicketGuilds, _resetTicketLookup, LOOKUP_REUSE_MS, MEMBER_PROBES_PER_DM } = + await import('../src/modules/tickets.js'); + +const SENDER = { id: '700000000000000002' }; +let nextId = 810000000000000000n; + +async function ticketGuild(fetches, { members = [] } = {}) { + const guild = { + id: String(nextId++), + client: {}, + // A tiny server whose cache holds only the bot: with the intent off this proves nothing. + memberCount: 2, + members: { + cache: new Map([['bot', {}]]), + fetch: async (id) => { + fetches.push(guild.id); + if (members.includes(id)) return { id }; + throw Object.assign(new Error('Unknown Member'), { code: 10007 }); + }, + }, + }; + await setGuildModule(guild.id, 'tickets', { enabled: true }); + return guild; +} + +test('intent off: a cache that looks complete is not trusted, the sender is found by fetching', async () => { + _resetTicketLookup(); + const fetches = []; + // memberCount 1 (only the bot) would look complete; here the sender is a second member. + const mine = await ticketGuild(fetches, { members: [SENDER.id] }); + mine.memberCount = 1; + mine.members.cache = new Map([['bot', {}]]); + const r = await lookupTicketGuilds(SENDER, { guilds: { cache: new Map([[mine.id, mine]]) } }); + assert.deepEqual( + r.guilds.map((g) => g.id), + [mine.id] + ); + assert.equal(fetches.length, 1); +}); + +test('intent off: a sender in server 13 of 14 is found within a few messages, with capped fetches', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 9_000_000 }); + const fetches = []; + const guilds = []; + for (let i = 0; i < 12; i++) guilds.push(await ticketGuild(fetches)); + const mine = await ticketGuild(fetches, { members: [SENDER.id] }); + guilds.push(mine); + guilds.push(await ticketGuild(fetches)); + const client = { guilds: { cache: new Map(guilds.map((g) => [g.id, g])) } }; + + const first = await lookupTicketGuilds(SENDER, client); + assert.equal(fetches.length, MEMBER_PROBES_PER_DM); + assert.equal(first.guilds.length, 0); + assert.equal(first.incomplete, true); + + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + const second = await lookupTicketGuilds(SENDER, client); + assert.deepEqual( + second.guilds.map((g) => g.id), + [mine.id] + ); + assert.ok(fetches.length <= MEMBER_PROBES_PER_DM * 2); +}); diff --git a/test/ticketGuildLookup.test.js b/test/ticketGuildLookup.test.js new file mode 100644 index 00000000..da922bb8 --- /dev/null +++ b/test/ticketGuildLookup.test.js @@ -0,0 +1,226 @@ +// A DM to the bot must not cost a member fetch in every ticket server of the instance. +import './helpers/tmpDb.js'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { setGuildModule } from '../src/db/modules.js'; +import { + lookupTicketGuilds, + canOpenTicketIn, + _resetTicketLookup, + MEMBER_PROBES_PER_DM, + MEMBER_PROBES_PER_MINUTE, + LOOKUP_REUSE_MS, +} from '../src/modules/tickets.js'; +import { register } from '../src/bot/events/dmTickets.js'; + +const SENDER = { id: '700000000000000001' }; +let nextId = 800000000000000000n; + +/** + * A ticket-enabled fake guild. `complete`: the member cache holds every member (so a miss means + * "not a member"). `members`: ids that really are members (a REST fetch finds them). + */ +async function ticketGuild(fetches, { complete = false, members = [], cached = [] } = {}) { + const guild = { + id: String(nextId++), + client: {}, + memberCount: complete ? 3 : 5000, + members: { + cache: new Map(cached.map((id) => [id, { id, partial: false }])), + fetch: async (id) => { + fetches.push({ guild: guild.id, id }); + if (members.includes(id)) return { id }; + throw Object.assign(new Error('Unknown Member'), { code: 10007 }); + }, + }, + }; + if (complete) for (let i = guild.members.cache.size; i < 3; i++) guild.members.cache.set(`x${i}`, {}); + await setGuildModule(guild.id, 'tickets', { enabled: true }); + return guild; +} + +const clientOf = (guilds) => ({ guilds: { cache: new Map(guilds.map((g) => [g.id, g])) } }); + +test('a sender in no server: complete caches answer without any REST fetch', async () => { + _resetTicketLookup(); + const fetches = []; + const guilds = []; + for (let i = 0; i < 30; i++) guilds.push(await ticketGuild(fetches, { complete: true })); + const r = await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.deepEqual(r, { guilds: [], incomplete: false }); + assert.equal(fetches.length, 0); +}); + +test('a cached member is found without a fetch, in the right server only', async () => { + _resetTicketLookup(); + const fetches = []; + const a = await ticketGuild(fetches, { cached: [SENDER.id] }); + const guilds = [a]; + for (let i = 0; i < 20; i++) guilds.push(await ticketGuild(fetches, { complete: true })); + const r = await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.deepEqual( + r.guilds.map((g) => g.id), + [a.id] + ); + assert.equal(fetches.length, 0); +}); + +test('a cached member of a server with tickets off is not offered', async () => { + _resetTicketLookup(); + const fetches = []; + const g = await ticketGuild(fetches, { cached: [SENDER.id] }); + await setGuildModule(g.id, 'tickets', { enabled: false }); + assert.deepEqual((await lookupTicketGuilds(SENDER, clientOf([g]))).guilds, []); +}); + +test('incomplete caches: fetches per DM are capped, remembered, and continue on the next DM', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 1_000_000 }); + const fetches = []; + const guilds = []; + for (let i = 0; i < 20; i++) guilds.push(await ticketGuild(fetches)); + const client = clientOf(guilds); + + const first = await lookupTicketGuilds(SENDER, client); + assert.equal(fetches.length, MEMBER_PROBES_PER_DM); + assert.deepEqual(first, { guilds: [], incomplete: true }); + + // Sending again at once reuses the answer: no new fetches (per-user cooldown). + await lookupTicketGuilds(SENDER, client); + assert.equal(fetches.length, MEMBER_PROBES_PER_DM); + + // Later, the next servers are checked; the ones already known as "not a member" are skipped. + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + await lookupTicketGuilds(SENDER, client); + assert.equal(fetches.length, MEMBER_PROBES_PER_DM * 2); + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + const last = await lookupTicketGuilds(SENDER, client); + assert.equal(fetches.length, 20, 'each server was fetched exactly once'); + assert.equal(last.incomplete, false); + assert.equal(new Set(fetches.map((f) => f.guild)).size, 20); +}); + +test('a real member in a late server is found, and found first the next time', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 2_000_000 }); + const fetches = []; + const guilds = []; + for (let i = 0; i < 12; i++) guilds.push(await ticketGuild(fetches)); + const mine = await ticketGuild(fetches, { members: [SENDER.id] }); + guilds.push(mine); + const client = clientOf(guilds); + + await lookupTicketGuilds(SENDER, client); // first 8 servers, not found + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + const r = await lookupTicketGuilds(SENDER, client); + assert.deepEqual( + r.guilds.map((g) => g.id), + [mine.id] + ); + + // Hint: the next lookup (everything forgotten except the hint) tries that server first. + t.mock.timers.tick(LOOKUP_REUSE_MS + 3 * 60_000); + fetches.length = 0; + const again = await lookupTicketGuilds(SENDER, client); + assert.equal(fetches[0].guild, mine.id); + assert.deepEqual( + again.guilds.map((g) => g.id), + [mine.id] + ); +}); + +test('all DMs together stay within the per-minute fetch budget', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 3_000_000 }); + const fetches = []; + const guilds = []; + for (let i = 0; i < 20; i++) guilds.push(await ticketGuild(fetches)); + const client = clientOf(guilds); + for (let u = 0; u < 40; u++) + await lookupTicketGuilds({ id: `71000000000000${String(u).padStart(4, '0')}` }, client); + assert.equal(fetches.length, MEMBER_PROBES_PER_MINUTE); + const late = await lookupTicketGuilds({ id: '720000000000000000' }, client); + assert.equal(late.incomplete, true); + t.mock.timers.tick(61_000); + await lookupTicketGuilds({ id: '720000000000000001' }, client); + assert.ok(fetches.length > MEMBER_PROBES_PER_MINUTE, 'the budget resets after a minute'); +}); + +test('a rate limit error is not remembered as "not a member"', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 4_000_000 }); + const g = await ticketGuild([]); + let calls = 0; + g.members.fetch = async (id) => { + calls++; + if (calls === 1) throw Object.assign(new Error('rate limited'), { status: 429 }); + return { id }; + }; + const first = await lookupTicketGuilds(SENDER, clientOf([g])); + assert.equal(first.incomplete, true); + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + const second = await lookupTicketGuilds(SENDER, clientOf([g])); + assert.equal(second.guilds.length, 1); +}); + +test('canOpenTicketIn: cache, or the lookup that just ran, or one fetch', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 5_000_000 }); + const fetches = []; + const cachedIn = await ticketGuild(fetches, { cached: [SENDER.id] }); + const memberOf = await ticketGuild(fetches, { members: [SENDER.id] }); + const notIn = await ticketGuild(fetches); + const completeNot = await ticketGuild(fetches, { complete: true }); + assert.equal(await canOpenTicketIn(SENDER, cachedIn), true); + assert.equal(fetches.length, 0); + assert.equal(await canOpenTicketIn(SENDER, completeNot), false); + assert.equal(fetches.length, 0); + assert.equal(await canOpenTicketIn(SENDER, memberOf), true); + assert.equal(await canOpenTicketIn(SENDER, notIn), false); + assert.equal(fetches.length, 2); + await setGuildModule(cachedIn.id, 'tickets', { enabled: false }); + assert.equal(await canOpenTicketIn(SENDER, cachedIn), false); +}); + +test('the DM handler asks the sender to try again when the lookup was cut short', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 6_000_000 }); + const guilds = []; + for (let i = 0; i < 12; i++) guilds.push(await ticketGuild([])); + const replies = []; + const message = { + guild: null, + partial: false, + content: 'I need help', + author: { id: '730000000000000001', bot: false }, + attachments: new Map(), + client: { ...clientOf(guilds), user: { id: '555555555555555555' } }, + reply: async (text) => replies.push(text), + react: async () => {}, + }; + const client = new EventEmitter(); + register(client); + const settle = () => new Promise((r) => setImmediate(r)); + client.emit('messageCreate', message); + for (let i = 0; i < 20; i++) await settle(); + assert.equal(replies.length, 1); + assert.match(replies[0], /could not check all servers/); + + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + client.emit('messageCreate', message); + for (let i = 0; i < 20; i++) await settle(); + assert.equal(replies.length, 2); + assert.match(replies[1], /not set up/, 'all servers checked by the second DM: a definite answer'); +}); + +test('a platform that answers null for a non-member (and has no member cache) is handled', async () => { + _resetTicketLookup(); + const g = await ticketGuild([]); + g.members = { fetch: async (id) => (id === SENDER.id ? null : { id }) }; + const r = await lookupTicketGuilds(SENDER, clientOf([g])); + assert.deepEqual(r, { guilds: [], incomplete: false }); + const mine = await ticketGuild([]); + mine.members = { fetch: async (id) => ({ id }) }; + assert.equal((await lookupTicketGuilds({ id: '700000000000000077' }, clientOf([mine]))).guilds.length, 1); +}); From 7c6e243b28e278f4dd4a89c7ad68a3823f78746c Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 00:53:55 +0100 Subject: [PATCH 03/21] fix(events): give each server a fair share of the event image cache The public event page image cache was one shared pool, so a server with more large images than fit could evict every other server's images and make each page view download again. Entries now belong to the server whose page asked for them: a server holds at most 4 MiB and 16 entries, its own oldest entries go first, and when the whole cache is full the biggest holder gives way. A server may start at most 20 downloads a minute and the instance 8 at once; a request over a limit gets no image and starts no download. Failures were already remembered and concurrent requests for one image already shared one download. --- src/modules/lib/eventImage.js | 109 +++++++++++++++++++---- src/web/routes/events.js | 9 +- test/eventImage.fairShare.test.js | 130 ++++++++++++++++++++++++++++ test/eventPages.imageBudget.test.js | 76 ++++++++++++++++ 4 files changed, 305 insertions(+), 19 deletions(-) create mode 100644 test/eventImage.fairShare.test.js create mode 100644 test/eventPages.imageBudget.test.js diff --git a/src/modules/lib/eventImage.js b/src/modules/lib/eventImage.js index e3a2a604..5ac276d7 100644 --- a/src/modules/lib/eventImage.js +++ b/src/modules/lib/eventImage.js @@ -15,7 +15,17 @@ // // Answers are kept in memory for a short while so a page that is opened often does not make Sylo // fetch the same image every time. The memory is bounded: at most MAX_ENTRIES entries and -// MAX_TOTAL_BYTES of image data, oldest out first; a failure is remembered for a shorter time. +// MAX_TOTAL_BYTES of image data; a failure is remembered for a shorter time. +// +// One server must not be able to use all of it, or to make Sylo download over and over. Every entry +// belongs to the server (the `owner`) whose page asked for it: +// - a server may hold at most MAX_OWNER_BYTES and MAX_OWNER_ENTRIES; past that its own oldest +// entries go, never another server's; +// - when the whole memory is full, the entries of the server that holds the most go first; +// - a server may start at most OWNER_FETCHES_PER_MINUTE downloads a minute (a failure counts), and +// the whole instance at most MAX_CONCURRENT_DOWNLOADS at once. A request over those limits +// gets no image (the page leaves it out) and starts no download. +// So a server with more large images than its share only ever refetches its own, at a bounded rate. // An entry that has passed its time is also removed, not just ignored: a sweep runs every // SWEEP_MS (and on every use), so nothing stays in memory more than about a minute past its time. import { safeFetch } from '../../lib/safeFetch.js'; @@ -29,6 +39,12 @@ export const FAILED_TTL_MS = 60_000; export const SWEEP_MS = 60_000; export const MAX_ENTRIES = 64; export const MAX_TOTAL_BYTES = 16 * 1024 * 1024; +/** One server's share of the memory. */ +export const MAX_OWNER_BYTES = 4 * 1024 * 1024; +export const MAX_OWNER_ENTRIES = 16; +/** Downloads one server may start per minute, and downloads running at once for all servers. */ +export const OWNER_FETCHES_PER_MINUTE = 20; +export const MAX_CONCURRENT_DOWNLOADS = 8; /** The types Sylo serves: what the bytes of the file must show. */ const SIGNATURES = [ @@ -61,8 +77,11 @@ export function _setImageFetcher(fn) { fetcher = fn ?? safeFetch; } -/** @type {Map} */ +/** @type {Map} */ const cache = new Map(); +/** owner -> { start, count }: downloads started in the current minute. */ +const fetchWindows = new Map(); +let running = 0; /** @type {Map>} */ const inflight = new Map(); let cachedBytes = 0; @@ -71,6 +90,7 @@ let cachedBytes = 0; export function _clearImageCache() { cache.clear(); inflight.clear(); + fetchWindows.clear(); cachedBytes = 0; } /** Tests only: how much is held. */ @@ -85,27 +105,77 @@ export function sweepImageCache(now = Date.now()) { cachedBytes -= entry.body?.length ?? 0; cache.delete(key); } + for (const [owner, w] of fetchWindows) if (now - w.start >= 60_000) fetchWindows.delete(owner); } every('events', SWEEP_MS, () => sweepImageCache()); +/** What one owner holds. */ +function usage(owner) { + let entries = 0; + let bytes = 0; + for (const e of cache.values()) { + if (e.owner !== owner) continue; + entries += 1; + bytes += e.body?.length ?? 0; + } + return { entries, bytes }; +} + +function evict(key) { + cachedBytes -= cache.get(key)?.body?.length ?? 0; + cache.delete(key); +} + +/** The oldest entry of an owner other than `keep`, or undefined. */ +function oldestOf(owner, keep) { + for (const [key, e] of cache) if (e.owner === owner && key !== keep) return key; + return undefined; +} + function remember(url, entry) { sweepImageCache(); - const old = cache.get(url); - if (old) { - cachedBytes -= old.body?.length ?? 0; - cache.delete(url); - } + if (cache.has(url)) evict(url); cache.set(url, entry); cachedBytes += entry.body?.length ?? 0; - // Oldest out first (a Map keeps insertion order) until both bounds hold. - for (const key of cache.keys()) { - if (cache.size <= MAX_ENTRIES && cachedBytes <= MAX_TOTAL_BYTES) break; - if (key === url) continue; - cachedBytes -= cache.get(key).body?.length ?? 0; - cache.delete(key); + // The owner's own share first: its oldest entries go (a Map keeps insertion order). + for (;;) { + const u = usage(entry.owner); + if (u.entries <= MAX_OWNER_ENTRIES && u.bytes <= MAX_OWNER_BYTES) break; + const victim = oldestOf(entry.owner, url); + if (victim === undefined) break; + evict(victim); + } + // Then the whole memory: the biggest holder gives up its oldest entry first. + while (cache.size > MAX_ENTRIES || cachedBytes > MAX_TOTAL_BYTES) { + const held = new Map(); + for (const e of cache.values()) { + const h = held.get(e.owner) ?? { bytes: 0, entries: 0 }; + h.bytes += e.body?.length ?? 0; + h.entries += 1; + held.set(e.owner, h); + } + const order = [...held].sort((x, y) => y[1].bytes - x[1].bytes || y[1].entries - x[1].entries); + const victim = order.map(([owner]) => oldestOf(owner, url)).find((k) => k !== undefined); + if (victim === undefined) break; + evict(victim); } } +/** May this owner start another download now? Counts it when yes. */ +function mayDownload(owner) { + if (running >= MAX_CONCURRENT_DOWNLOADS) return false; + if (owner === null) return true; + const now = Date.now(); + let w = fetchWindows.get(owner); + if (!w || now - w.start >= 60_000) { + w = { start: now, count: 0 }; + fetchWindows.set(owner, w); + } + if (w.count >= OWNER_FETCHES_PER_MINUTE) return false; + w.count += 1; + return true; +} + async function download(url) { try { const res = await fetcher(url, { @@ -131,21 +201,28 @@ async function download(url) { /** * The image behind a link, or null when it may not be fetched or the fetch fails. * @param {unknown} link an address from the event or the server, never from a visitor + * @param {string | null} [owner] the server the page belongs to: its share of the memory and + * its download budget (see the top of the file); null for none * @returns {Promise<{ type: string, body: Buffer } | null>} */ -export async function loadImage(link) { +export async function loadImage(link, owner = null) { const url = trustedImageUrl(link); if (!url) return null; sweepImageCache(); const hit = cache.get(url); if (hit) return hit.body ? { type: hit.type, body: hit.body } : null; if (inflight.has(url)) return inflight.get(url); + if (!mayDownload(owner)) return null; // over the limits: no image now, and nothing is started + running += 1; const pending = download(url) .then((image) => { - remember(url, { at: Date.now(), type: image?.type ?? null, body: image?.body ?? null }); + remember(url, { at: Date.now(), owner, type: image?.type ?? null, body: image?.body ?? null }); return image; }) - .finally(() => inflight.delete(url)); + .finally(() => { + running -= 1; + inflight.delete(url); + }); inflight.set(url, pending); return pending; } diff --git a/src/web/routes/events.js b/src/web/routes/events.js index 62d0a9dc..cfbba390 100644 --- a/src/web/routes/events.js +++ b/src/web/routes/events.js @@ -181,8 +181,8 @@ router.get('/:token', async (req, res, next) => { const [going, maybe, icon, image] = await Promise.all([ eventStatusCount(ev.id, 'join'), eventStatusCount(ev.id, 'maybe'), - loadImage(iconLink(guild)), - loadImage(ev.image_url), + loadImage(iconLink(guild), ev.guild_id), + loadImage(ev.image_url, ev.guild_id), ]); const status = eventStatus(ev); // The posted time changes as people edit; the page must not be served from a cache after the @@ -221,7 +221,10 @@ router.get('/:token/:asset', async (req, res, next) => { if (asset !== 'icon' && asset !== 'image') return notFound(res); const open = await openEvent(req); if (open.type !== 'page') return refuse(res, open, token); - const image = await loadImage(asset === 'icon' ? iconLink(open.guild) : open.ev.image_url); + const image = await loadImage( + asset === 'icon' ? iconLink(open.guild) : open.ev.image_url, + open.ev.guild_id + ); if (!image) return notFound(res); res.set({ 'Content-Type': image.type, diff --git a/test/eventImage.fairShare.test.js b/test/eventImage.fairShare.test.js new file mode 100644 index 00000000..f975ef93 --- /dev/null +++ b/test/eventImage.fairShare.test.js @@ -0,0 +1,130 @@ +// One server must not use all of the event image memory, or make Sylo download over and over +// (modules/lib/eventImage.js). +import './helpers/tmpDb.js'; +import test, { mock, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { + MAX_CONCURRENT_DOWNLOADS, + MAX_ENTRIES, + MAX_IMAGE_BYTES, + MAX_OWNER_BYTES, + MAX_OWNER_ENTRIES, + MAX_TOTAL_BYTES, + OWNER_FETCHES_PER_MINUTE, + _clearImageCache, + _imageCacheSize, + _setImageFetcher, + loadImage, +} from '../src/modules/lib/eventImage.js'; + +const PNG_HEAD = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const SMALL = Buffer.concat([PNG_HEAD, Buffer.from('x')]); +const BIG = Buffer.concat([PNG_HEAD, Buffer.alloc(MAX_IMAGE_BYTES - PNG_HEAD.length - 16)]); +const url = (name) => `https://cdn.discordapp.com/attachments/1/2/${name}.png`; +const reply = (body) => ({ + status: 200, + ok: true, + headers: new Headers({ 'content-type': 'image/png' }), + body, +}); + +let calls; +function fetcher(answer) { + calls = []; + _setImageFetcher(async (link) => { + calls.push(link); + return answer(link); + }); +} +afterEach(() => { + mock.restoreAll(); + _setImageFetcher(null); + _clearImageCache(); +}); + +test('limits are consistent: a server share is a fraction of the whole', () => { + assert.ok(MAX_OWNER_BYTES * 2 <= MAX_TOTAL_BYTES); + assert.ok(MAX_OWNER_ENTRIES * 2 <= MAX_ENTRIES); + assert.ok(MAX_OWNER_BYTES >= MAX_IMAGE_BYTES, 'one image of the biggest size fits in a share'); +}); + +test('a server with many big images cannot push another server image out', async () => { + fetcher((link) => reply(link.includes('small') ? SMALL : BIG)); + await loadImage(url('small-b'), 'guild-b'); + for (let n = 0; n < 9; n += 1) await loadImage(url(`big-a${n}`), 'guild-a'); + const before = calls.length; + await loadImage(url('small-b'), 'guild-b'); + assert.equal(calls.length, before, 'guild B image is still cached'); + assert.ok(_imageCacheSize().bytes <= MAX_OWNER_BYTES + SMALL.length, 'guild A holds only its share'); +}); + +test('when the whole memory is full, the biggest holder gives way first', async () => { + fetcher((link) => reply(link.includes('small') ? SMALL : BIG)); + // Four servers fill their share each (16 MiB in all), then a small server adds one image. + for (const owner of ['a', 'b', 'c', 'd']) { + for (let n = 0; n < 2; n += 1) await loadImage(url(`big-${owner}${n}`), owner); + } + await loadImage(url('small-e'), 'e'); + await loadImage(url('small-f'), 'f'); + assert.ok(_imageCacheSize().bytes <= MAX_TOTAL_BYTES); + const before = calls.length; + await loadImage(url('small-e'), 'e'); + await loadImage(url('small-f'), 'f'); + assert.equal(calls.length, before, 'the small servers images stayed'); +}); + +test('a refetch storm is bounded: one server starts at most OWNER_FETCHES_PER_MINUTE downloads a minute', async () => { + fetcher(() => reply(BIG)); + for (let round = 0; round < 5; round += 1) { + for (let n = 0; n < 9; n += 1) await loadImage(url(`storm${n}`), 'guild-a'); + } + assert.ok(calls.length <= OWNER_FETCHES_PER_MINUTE, `made ${calls.length} downloads`); + assert.equal(await loadImage(url('storm-new'), 'guild-a'), null, 'over the limit: no image, no download'); + assert.equal(calls.length, OWNER_FETCHES_PER_MINUTE); +}); + +test('the limit is per server, and it renews after a minute', async () => { + const now = mock.method(Date, 'now', () => 1_000_000); + fetcher(() => reply(SMALL)); + for (let n = 0; n < OWNER_FETCHES_PER_MINUTE + 5; n += 1) await loadImage(url(`a${n}`), 'guild-a'); + assert.equal(calls.length, OWNER_FETCHES_PER_MINUTE); + assert.ok(await loadImage(url('b0'), 'guild-b'), 'another server is not affected'); + now.mock.mockImplementation(() => 1_000_000 + 61_000); + assert.ok(await loadImage(url('a-later'), 'guild-a'), 'a minute later it may download again'); +}); + +test('a failure is remembered, so a broken link costs one download', async () => { + fetcher(async () => ({ status: 404, ok: false, headers: new Headers(), body: Buffer.alloc(0) })); + for (let n = 0; n < 10; n += 1) assert.equal(await loadImage(url('gone'), 'guild-a'), null); + assert.equal(calls.length, 1); +}); + +test('visitors who arrive together share one download, and it counts once', async () => { + let release; + const gate = new Promise((r) => (release = r)); + fetcher(async () => { + await gate; + return reply(SMALL); + }); + const all = Promise.all(Array.from({ length: 30 }, () => loadImage(url('same'), 'guild-a'))); + release(); + await all; + assert.equal(calls.length, 1); +}); + +test('at most MAX_CONCURRENT_DOWNLOADS run at once across servers', async () => { + let release; + const gate = new Promise((r) => (release = r)); + fetcher(async () => { + await gate; + return reply(SMALL); + }); + const loads = Array.from({ length: MAX_CONCURRENT_DOWNLOADS + 3 }, (_, n) => + loadImage(url(`c${n}`), `guild-${n}`) + ); + assert.equal(calls.length, MAX_CONCURRENT_DOWNLOADS); + release(); + const results = await Promise.all(loads); + assert.equal(results.filter(Boolean).length, MAX_CONCURRENT_DOWNLOADS); + assert.ok(await loadImage(url('after'), 'guild-x'), 'the slots are free again'); +}); diff --git a/test/eventPages.imageBudget.test.js b/test/eventPages.imageBudget.test.js new file mode 100644 index 00000000..18983de5 --- /dev/null +++ b/test/eventPages.imageBudget.test.js @@ -0,0 +1,76 @@ +// Anonymous views of public event pages must not turn into an endless stream of image downloads, +// even when a server's events hold more big images than its share of the cache. +import './helpers/trustLoopbackProxy.js'; +import { startWebApp } from './helpers/webApp.js'; +import { CH, GID } from './helpers/fakeGuild.js'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { setGuildModule } from '../src/db/modules.js'; +import { createEvent } from '../src/db/events.js'; +import { + MAX_IMAGE_BYTES, + OWNER_FETCHES_PER_MINUTE, + _clearImageCache, + _setImageFetcher, +} from '../src/modules/lib/eventImage.js'; + +const SG = '900000000000000126'; +const PNG_HEAD = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const BIG = Buffer.concat([PNG_HEAD, Buffer.alloc(MAX_IMAGE_BYTES - PNG_HEAD.length - 16)]); +const ICON_LINK = `https://cdn.discordapp.com/icons/${GID}/a1b2c3.png?size=128`; + +let app; +const calls = []; +test.before(async () => { + app = await startWebApp(); + app.guild.iconURL = () => ICON_LINK; + app.client.guilds.cache.set(SG, app.guild); + _setImageFetcher(async (link) => { + calls.push(link); + return { status: 200, ok: true, headers: new Headers({ 'content-type': 'image/png' }), body: BIG }; + }); + _clearImageCache(); + await setGuildModule(SG, 'events', { enabled: true, config: { publicPages: true } }); +}); +test.after(async () => { + _setImageFetcher(null); + await app.close(); +}); + +test('27 page views over 9 big-image events start a bounded number of downloads', async () => { + const events = []; + for (let n = 0; n < 9; n += 1) { + const { id } = await createEvent({ + guildId: SG, + channelId: CH.general, + title: `Night ${n}`, + description: 'Bring snacks', + location: 'Voice 1', + capacity: 10, + roleId: '100000000000000700', + hostId: '900000000000000555', + startAt: Date.UTC(2030, 0, 5, 18, 0), + endAt: Date.UTC(2030, 0, 5, 21, 0), + timeZone: 'Europe/Oslo', + imageUrl: `https://cdn.discordapp.com/attachments/${CH.general}/100000000000000042/p${n}.png`, + }); + events.push(id); + } + const { getEvent } = await import('../src/db/events.js'); + let views = 0; + for (let round = 0; round < 3; round += 1) { + for (const id of events) { + const ev = await getEvent(id); + const res = await fetch(`${app.base}/events/${ev.public_token}`, { + headers: { 'x-forwarded-for': `10.30.${views++}.1` }, + }); + assert.equal(res.status, 200); + await res.arrayBuffer(); + } + } + assert.equal(views, 27); + assert.ok( + calls.length <= OWNER_FETCHES_PER_MINUTE, + `${calls.length} downloads for 27 views (limit ${OWNER_FETCHES_PER_MINUTE})` + ); +}); From f3e88641694dde0c3931d3a6ee14e44b6a21fb99 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 00:59:55 +0100 Subject: [PATCH 04/21] feat(backup): warn about a privileged database role and confirm before restoring an uploaded dump pg_restore runs every statement of a restored dump as the database role in DATABASE_URL. Sylo cannot change that role, so: - at start-up on Postgres it reads the role and logs one warning when it is a superuser or has CREATEROLE, CREATEDB or one of the pg_execute_server_program, pg_read_server_files and pg_write_server_files memberships, with a link to the docs; - /api/v2/health reports the same risks (db.roleRisks, db.roleDocsUrl) and the Health page shows a note; - restoring an uploaded (imported) dump asks for confirmation that says its SQL runs with the rights of the database role, and importing a dump on Postgres says the same. Restores are not blocked; - docs/postgres.md describes a least-privilege role setup and how to check one. --- docs/postgres.md | 53 ++++++++++++- docs/self-hosting.md | 2 +- src/db/dbRole.js | 107 +++++++++++++++++++++++++++ src/index.js | 3 + src/web/routes/v2Api.js | 4 + test/dbRole.postgres.test.js | 39 ++++++++++ test/dbRole.test.js | 56 ++++++++++++++ web-v2/src/__tests__/Health.test.jsx | 60 ++++++++++++++- web-v2/src/pages/Health.jsx | 52 ++++++++++++- 9 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 src/db/dbRole.js create mode 100644 test/dbRole.postgres.test.js create mode 100644 test/dbRole.test.js diff --git a/docs/postgres.md b/docs/postgres.md index be25ffce..4e650535 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -9,6 +9,7 @@ be a real bottleneck. If that's not you, skip this page — - [Should you use this?](#should-you-use-this) - [Enabling Postgres for a new install](#enabling-postgres-for-a-new-install) - [Moving an existing SQLite install over](#moving-an-existing-sqlite-install-over) +- [Least-privilege database role](#least-privilege-database-role) - [Backups and restore on Postgres](#backups-and-restore-on-postgres) - [Rolling back](#rolling-back) - [Troubleshooting](#troubleshooting) @@ -35,7 +36,8 @@ Set `DATABASE_URL` before the first start: DATABASE_URL=postgres://user:pass@host:5432/dbname ``` -That's the whole setup. On first connect, Sylo creates every table it needs +Use a database role with few rights for this, not a superuser: see +[Least-privilege database role](#least-privilege-database-role). That's the whole setup. On first connect, Sylo creates every table it needs automatically — no separate schema step, no migration to run by hand. Leave `DATABASE_URL` unset (the default) and nothing changes: Sylo uses `DATABASE_PATH` exactly as it always has. @@ -121,6 +123,51 @@ giveaway's entries) stay intact — confirmed by inserting a new row through Sylo's own code right after a migration and checking it didn't collide with a copied one. +## Least-privilege database role + +The role in `DATABASE_URL` should own the Sylo database and have no other rights. The reason is +backup import. A Postgres dump is a file of SQL, and **restoring one (including a file an operator +uploaded on the Health page) runs that SQL with the rights of the role Sylo connects as**: +`pg_restore` sends every statement in the file to the database as that role. Inside Sylo's own +database that adds nothing an operator cannot already do, because a restore replaces the whole +database. With a superuser role, or a role that is a member of `pg_execute_server_program`, +`pg_read_server_files` or `pg_write_server_files`, the SQL can also reach the database server's +own system (`COPY ... PROGRAM`, server files). Sylo cannot change the role from code, so set it up +once when you create the database. At start-up Sylo reads the role and logs one warning if it has +`SUPERUSER`, `CREATEROLE`, `CREATEDB` or one of those three memberships; the Health page shows the +same note. + +Create the database and the role as an administrator of the Postgres server (with the official +`postgres` image that is its `POSTGRES_USER`, which is a superuser: use it for this setup only, +never in `DATABASE_URL`): + +```sql +CREATE ROLE sylo LOGIN PASSWORD 'choose-a-long-random-password' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS; +CREATE DATABASE sylo OWNER sylo; +REVOKE ALL ON DATABASE sylo FROM PUBLIC; +-- then, connected to the new database (\c sylo): +ALTER SCHEMA public OWNER TO sylo; +``` + +Then point Sylo at it: `DATABASE_URL=postgres://sylo:choose-a-long-random-password@host:5432/sylo`. +Sylo creates its tables itself on first start, so the role owns them, and `pg_restore --clean` +can drop and recreate them without extra rights. To check an existing setup, connect with the same +credentials Sylo uses and run: + +```sql +SELECT rolsuper, rolcreaterole, rolcreatedb FROM pg_roles WHERE rolname = current_user; +SELECT pg_has_role(current_user, 'pg_execute_server_program', 'MEMBER') AS exec_program, + pg_has_role(current_user, 'pg_read_server_files', 'MEMBER') AS read_files, + pg_has_role(current_user, 'pg_write_server_files', 'MEMBER') AS write_files; +``` + +Every value should be `f`. Restores are never blocked because of the role; the Health page asks +you to confirm before it restores an uploaded dump and says that its SQL runs with the role's +rights. Even with a least-privilege role, a restored dump can still do anything that role may do +inside the Sylo database (create functions and triggers, change or delete any row), so restore only +files you made or trust. + ## Backups and restore on Postgres The Health page's backup/restore works the same way on both drivers — same @@ -139,6 +186,10 @@ restores onto the driver it came from — trying to restore a `.dump` while running on SQLite (or vice versa) fails with a clear error instead of doing something destructive. +A restore runs the SQL in the dump with the rights of the database role Sylo connects as, so +use the [least-privilege role](#least-privilege-database-role) above and restore only dumps you +made or trust. + `pg_dump`/`pg_restore` are bundled in the Docker image, pinned to the newest supported Postgres major version — that covers dumping from and restoring to any older server version too (a newer client is safe against an older diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 8e70e1b2..787c23e8 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -100,7 +100,7 @@ At least one platform must be configured. | `GAMETOOLS_API_BASE` | `https://api.gametools.network` | Stats API base URL | | `STATS_CACHE_TTL_MINUTES` | `5` | How long stats lookups are cached | | `DATABASE_PATH` | `./data/sylo.db` | SQLite file path | -| `DATABASE_URL` | — | Optional: a `postgres://` URL to use Postgres instead of SQLite. Hosted-scale deployments only — see [docs/postgres.md](postgres.md). Unset (default) means nothing about the setup below changes. | +| `DATABASE_URL` | — | Optional: a `postgres://` URL to use Postgres instead of SQLite. Hosted-scale deployments only — see [docs/postgres.md](postgres.md), and use a [role with few rights](postgres.md#least-privilege-database-role), not a superuser. Unset (default) means nothing about the setup below changes. | | `BACKUP_INTERVAL_HOURS` | `24` | Scheduled DB snapshot interval; `0` disables it (pre-migration + manual still run) | | `BACKUP_RETENTION` | `14` | How many DB snapshots to keep in `/backups` | | `BACKUP_DIR` | `/backups` | Where DB snapshots are written | diff --git a/src/db/dbRole.js b/src/db/dbRole.js new file mode 100644 index 00000000..647d0db9 --- /dev/null +++ b/src/db/dbRole.js @@ -0,0 +1,107 @@ +// What the Postgres role in DATABASE_URL may do. Sylo cannot change it from code, so at start-up it +// reads the role's privileges and warns once when the role is stronger than Sylo needs. +// +// Why it matters: a backup file that an operator imports is restored with pg_restore, which sends +// every statement in the file to the database as this role (src/db/backupPostgres.js). Inside +// Sylo's own database that is no more than an operator can already do by restoring a backup. A +// superuser, or a member of pg_execute_server_program / pg_read_server_files / +// pg_write_server_files, can also reach the database server's own system. The role Sylo uses +// should have none of that: docs/postgres.md (Least-privilege database role). +import { config } from '../config.js'; +import { prepare } from './driver.js'; + +export const ROLE_DOCS_URL = + 'https://github.com/Ferdinand99/Sylo/blob/main/docs/postgres.md#least-privilege-database-role'; + +/** + * @typedef {object} RolePrivileges + * @property {string} name + * @property {boolean} superuser + * @property {boolean} createRole + * @property {boolean} createDb + * @property {boolean} execProgram member of pg_execute_server_program + * @property {boolean} readFiles member of pg_read_server_files + * @property {boolean} writeFiles member of pg_write_server_files + */ + +const yes = (v) => v === true || v === 1 || v === 't'; + +/** The privileges that are more than Sylo needs, in words. Empty when the role is fine. */ +export function roleRisks(p) { + if (!p) return []; + const risks = []; + if (p.superuser) risks.push('SUPERUSER'); + if (p.createRole) risks.push('CREATEROLE'); + if (p.createDb) risks.push('CREATEDB'); + if (p.execProgram) risks.push('pg_execute_server_program'); + if (p.readFiles) risks.push('pg_read_server_files'); + if (p.writeFiles) risks.push('pg_write_server_files'); + return risks; +} + +/** The one start-up warning for these privileges, or null when there is nothing to warn about. */ +export function rolePrivilegeWarning(p) { + const risks = roleRisks(p); + if (risks.length === 0) return null; + return ( + `The database role "${p.name}" in DATABASE_URL has more rights than Sylo needs (${risks.join(', ')}). ` + + 'A backup file imported from the Health page is restored with the rights of this role: its SQL runs as it. ' + + `Use a role without these rights: ${ROLE_DOCS_URL}` + ); +} + +/** @type {RolePrivileges | null} */ +let last = null; + +/** The result of the last check, or null when none ran (SQLite, or the check failed). */ +export const lastRolePrivileges = () => last; + +const FULL = ` + SELECT current_user AS name, r.rolsuper, r.rolcreaterole, r.rolcreatedb, + pg_has_role(current_user, 'pg_execute_server_program', 'MEMBER') AS exec_program, + pg_has_role(current_user, 'pg_read_server_files', 'MEMBER') AS read_files, + pg_has_role(current_user, 'pg_write_server_files', 'MEMBER') AS write_files + FROM pg_roles r WHERE r.rolname = current_user`; +// Servers before Postgres 11 have no predefined roles to ask about. +const BASIC = ` + SELECT current_user AS name, r.rolsuper, r.rolcreaterole, r.rolcreatedb + FROM pg_roles r WHERE r.rolname = current_user`; + +/** + * Read the privileges of the connected role. Postgres only; resolves to null on SQLite or when the + * server cannot be asked. Never throws. + * @returns {Promise} + */ +export async function readRolePrivileges() { + if (!config.databaseUrl) return null; + for (const sql of [FULL, BASIC]) { + try { + const row = await prepare(sql).get(); + if (!row) return null; + return { + name: String(row.name), + superuser: yes(row.rolsuper), + createRole: yes(row.rolcreaterole), + createDb: yes(row.rolcreatedb), + execProgram: yes(row.exec_program), + readFiles: yes(row.read_files), + writeFiles: yes(row.write_files), + }; + } catch { + // try the simpler question + } + } + return null; +} + +/** + * Start-up check: read the role's privileges, remember them for the Health page, and log one + * warning when the role is stronger than needed. + * @param {{ warn: (scope: string, msg: string) => void }} log + */ +export async function checkDatabaseRole(log) { + last = await readRolePrivileges(); + const warning = rolePrivilegeWarning(last); + if (warning) log.warn('db', warning); + return last; +} diff --git a/src/index.js b/src/index.js index fa06199e..2dfd1b09 100644 --- a/src/index.js +++ b/src/index.js @@ -9,6 +9,7 @@ import { runtime } from './runtime.js'; // Importing db/index.js opens the SQLite connection and runs migrations. import { closeDb } from './db/index.js'; import { startBackupSchedule } from './db/backup.js'; +import { checkDatabaseRole } from './db/dbRole.js'; import { startRetentionSchedule } from './db/retention.js'; import { startBot } from './bot/index.js'; import { startFluxerBot } from './bot/fluxerBot.js'; @@ -105,6 +106,8 @@ async function main() { ); } warnIfDataFolderIsNotAVolume(); + // Postgres: one warning when the database role is stronger than Sylo needs (see db/dbRole.js). + await checkDatabaseRole(log); // Every configured platform starts in parallel; one that is not configured is never loaded. await Promise.all([config.discordEnabled && startBot(), config.fluxerEnabled && startFluxerBot()]); httpServer = await startWeb(config.webPort); diff --git a/src/web/routes/v2Api.js b/src/web/routes/v2Api.js index 1db30f44..a527004b 100644 --- a/src/web/routes/v2Api.js +++ b/src/web/routes/v2Api.js @@ -331,6 +331,7 @@ import { isBackupName, inspectDbFile, } from '../../db/backup.js'; +import { roleRisks, lastRolePrivileges, ROLE_DOCS_URL } from '../../db/dbRole.js'; import { offsiteBackupStatus } from '../../db/offsiteBackup.js'; import { MODULES, getModule } from '../../modules/registry.js'; import { timeAgo, formatUptime, formatBytes } from '../lib/format.js'; @@ -2800,6 +2801,9 @@ router.get( size: formatBytes(dbInfo.size), wal: dbInfo.wal === null ? null : formatBytes(dbInfo.wal), postgres: Boolean(config.databaseUrl), + // Rights of the database role that are more than Sylo needs (empty when fine, or on SQLite). + roleRisks: roleRisks(lastRolePrivileges()), + roleDocsUrl: ROLE_DOCS_URL, intervalHours: config.backupIntervalHours, retention: config.backupRetention, offsite: offsiteBackupStatus(), diff --git a/test/dbRole.postgres.test.js b/test/dbRole.postgres.test.js new file mode 100644 index 00000000..448a2b96 --- /dev/null +++ b/test/dbRole.postgres.test.js @@ -0,0 +1,39 @@ +// The role check against a real Postgres connection. Skipped without DATABASE_URL. +import './helpers/isolateSqlite.js'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { closePostgres, prepare } from '../src/db/driver.js'; + +const url = process.env.DATABASE_URL; + +test( + 'dbRole reads the privileges of the connected role and warns once', + { skip: !url && 'DATABASE_URL not set (sqlite-only run)' }, + async (t) => { + const { checkDatabaseRole, readRolePrivileges, lastRolePrivileges, roleRisks } = + await import('../src/db/dbRole.js'); + t.after(async () => { + await closePostgres(); + }); + + const direct = await prepare( + 'SELECT current_user AS name, rolsuper, rolcreaterole, rolcreatedb FROM pg_roles WHERE rolname = current_user' + ).get(); + const read = await readRolePrivileges(); + assert.equal(read.name, direct.name); + assert.equal(read.superuser, direct.rolsuper === true); + assert.equal(read.createRole, direct.rolcreaterole === true); + assert.equal(read.createDb, direct.rolcreatedb === true); + for (const key of ['execProgram', 'readFiles', 'writeFiles']) assert.equal(typeof read[key], 'boolean'); + if (read.superuser) { + // A superuser is a member of every predefined role. + assert.deepEqual([read.execProgram, read.readFiles, read.writeFiles], [true, true, true]); + } + + const warned = []; + await checkDatabaseRole({ warn: (scope, msg) => warned.push([scope, msg]) }); + assert.equal(lastRolePrivileges().name, read.name); + assert.equal(warned.length, roleRisks(read).length > 0 ? 1 : 0); + if (warned.length) assert.match(warned[0][1], /docs\/postgres\.md#least-privilege-database-role/); + } +); diff --git a/test/dbRole.test.js b/test/dbRole.test.js new file mode 100644 index 00000000..6bfc2512 --- /dev/null +++ b/test/dbRole.test.js @@ -0,0 +1,56 @@ +// The Postgres role in DATABASE_URL: what counts as more than Sylo needs, and the one warning. +import './helpers/tmpDb.js'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + ROLE_DOCS_URL, + checkDatabaseRole, + readRolePrivileges, + rolePrivilegeWarning, + roleRisks, +} from '../src/db/dbRole.js'; + +const plain = { + name: 'sylo', + superuser: false, + createRole: false, + createDb: false, + execProgram: false, + readFiles: false, + writeFiles: false, +}; + +test('a role with no extra rights gets no warning', () => { + assert.deepEqual(roleRisks(plain), []); + assert.equal(rolePrivilegeWarning(plain), null); + assert.equal(rolePrivilegeWarning(null), null); +}); + +test('superuser, CREATEROLE and CREATEDB each warn, naming the right and linking the docs', () => { + for (const [key, label] of [ + ['superuser', 'SUPERUSER'], + ['createRole', 'CREATEROLE'], + ['createDb', 'CREATEDB'], + ['execProgram', 'pg_execute_server_program'], + ['readFiles', 'pg_read_server_files'], + ['writeFiles', 'pg_write_server_files'], + ]) { + const warning = rolePrivilegeWarning({ ...plain, [key]: true }); + assert.match(warning, new RegExp(label), key); + assert.match(warning, /"sylo"/); + assert.ok(warning.includes(ROLE_DOCS_URL)); + assert.match(warning, /imported from the Health page/); + } + assert.deepEqual(roleRisks({ ...plain, superuser: true, createDb: true }), ['SUPERUSER', 'CREATEDB']); +}); + +test('the docs link points at the least-privilege section of docs/postgres.md', () => { + assert.match(ROLE_DOCS_URL, /docs\/postgres\.md#least-privilege-database-role$/); +}); + +test('on SQLite there is nothing to read and nothing to warn about', async () => { + const warned = []; + assert.equal(await readRolePrivileges(), null); + assert.equal(await checkDatabaseRole({ warn: (...a) => warned.push(a) }), null); + assert.deepEqual(warned, []); +}); diff --git a/web-v2/src/__tests__/Health.test.jsx b/web-v2/src/__tests__/Health.test.jsx index 96f35198..8d60bf61 100644 --- a/web-v2/src/__tests__/Health.test.jsx +++ b/web-v2/src/__tests__/Health.test.jsx @@ -1,7 +1,9 @@ -import { render, screen } from '@testing-library/react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import Health from '../pages/Health.jsx'; +vi.mock('../components/ConfirmDialog.jsx', () => ({ confirmDialog: vi.fn().mockResolvedValue(false) })); + vi.mock('../api.js', () => ({ getHealth: vi.fn(), sendDevLogTest: vi.fn(), @@ -14,6 +16,7 @@ vi.mock('../api.js', () => ({ ApiError: class ApiError extends Error {}, })); const { getHealth } = await import('../api.js'); +const { confirmDialog } = await import('../components/ConfirmDialog.jsx'); function health(sessionSecret) { return { @@ -84,3 +87,58 @@ describe('Health: session secret warning', () => { expect(screen.queryByText(/session secret/i)).toBeNull(); }); }); + +describe('Health: restoring an uploaded Postgres dump', () => { + beforeEach(() => vi.clearAllMocks()); + + const pgHealth = (db = {}) => ({ + ...health(), + db: { + ...health().db, + postgres: true, + roleRisks: [], + roleDocsUrl: 'https://example.test/docs/postgres.md#least-privilege-database-role', + ...db, + }, + backups: [ + { name: 'sylo-imported-20261116.dump', size: '1 MB', ago: '1m ago' }, + { name: 'sylo-20261116.dump', size: '1 MB', ago: '2m ago' }, + ], + }); + const restoreButtons = () => screen.getAllByRole('button', { name: 'Restore' }); + const dialogBody = (call) => render(
{confirmDialog.mock.calls[call][0].body}
); + + it('says the dump runs with the database role rights before an imported dump is restored', async () => { + getHealth.mockResolvedValue(pgHealth({ roleRisks: ['SUPERUSER'] })); + render(); + await screen.findByText('sylo-imported-20261116.dump'); + fireEvent.click(restoreButtons()[0]); + await waitFor(() => expect(confirmDialog).toHaveBeenCalledTimes(1)); + const { container } = dialogBody(0); + expect(container.textContent).toMatch(/This file was uploaded/); + expect(container.textContent).toMatch(/runs that SQL with the rights of the database role/); + expect(container.textContent).toMatch(/SUPERUSER/); + expect(container.querySelector('a')?.getAttribute('href')).toMatch(/least-privilege-database-role/); + }); + + it('does not add that warning for a backup Sylo made itself', async () => { + getHealth.mockResolvedValue(pgHealth()); + render(); + await screen.findByText('sylo-20261116.dump'); + fireEvent.click(restoreButtons()[1]); + await waitFor(() => expect(confirmDialog).toHaveBeenCalledTimes(1)); + expect(dialogBody(0).container.textContent).not.toMatch(/This file was uploaded/); + }); + + it('shows a note on the page when the role is stronger than needed, and not otherwise', async () => { + getHealth.mockResolvedValue(pgHealth({ roleRisks: ['SUPERUSER', 'CREATEDB'] })); + const first = render(); + expect(await screen.findByText(/more rights than Sylo needs/i)).toBeTruthy(); + first.unmount(); + + getHealth.mockResolvedValue(pgHealth()); + render(); + await screen.findByText('Dev-log channel'); + expect(screen.queryByText(/more rights than Sylo needs/i)).toBeNull(); + }); +}); diff --git a/web-v2/src/pages/Health.jsx b/web-v2/src/pages/Health.jsx index fbd7b537..f74c2b71 100644 --- a/web-v2/src/pages/Health.jsx +++ b/web-v2/src/pages/Health.jsx @@ -15,6 +15,7 @@ import { confirmDialog } from '../components/ConfirmDialog.jsx'; import { plural } from '../components/Meta.jsx'; import { formatCount } from '../util.js'; import Reveal from '../components/Reveal.jsx'; +import Callout from '../components/Callout.jsx'; import { Icon } from '../components/icons.jsx'; // Plain names for the error sources that are not a module (a module's own name comes from the server). @@ -75,6 +76,28 @@ function Sparkline({ history }) { ); } +/** The name an imported backup is stored under (src/db/backup.js). */ +export const isImportedBackup = (name) => /^sylo-imported-/.test(name); + +/** + * Said before an uploaded Postgres dump is restored: the SQL in the file is run by the database + * with the rights of the role Sylo connects as. + */ +function UploadedSqlWarning({ risks = [], docsUrl }) { + return ( +

+ This file was uploaded. A Postgres dump holds SQL, and restoring it runs that SQL with + the rights of the database role Sylo connects as + {risks.length ? <> (this role has {risks.join(', ')}) : null}. Restore only a file you made or trust.{' '} + {docsUrl ? ( + + Use a role with few rights + + ) : null} +

+ ); +} + export default function Health() { const [state, setState] = useState({ loading: true, data: null, error: null }); const [notice, setNotice] = useState(null); @@ -174,8 +197,13 @@ export default function Health() { title: 'Restore the database?', body: ( <> - All current data is replaced with {name} and the bot restarts. - A prerestore snapshot is taken first. +

+ All current data is replaced with {name} and the bot + restarts. A prerestore snapshot is taken first. +

+ {d.db.postgres && isImportedBackup(name) ? ( + + ) : null} ), confirmLabel: 'Restore and restart', @@ -199,6 +227,13 @@ export default function Health() { <> {file.name} is stored as a snapshot. Nothing is restored until you press Restore on its row. + {d.db.postgres ? ( + <> + {' '} + Restoring a Postgres dump runs the SQL in it with the rights of Sylo's database role, so import + only files you made or trust. + + ) : null} ), confirmLabel: 'Import', @@ -441,6 +476,19 @@ export default function Health() { {fileName ? null : Choose a file first.} + {d.db.postgres && d.db.roleRisks?.length ? ( + +

+ The role Sylo connects as has {d.db.roleRisks.join(', ')}. A restored dump runs its SQL with + those rights.{' '} + {d.db.roleDocsUrl ? ( + + How to set up a role with few rights + + ) : null} +

+
+ ) : null}

Importing stores the file as a snapshot; you then Restore it. Restoring replaces the live database and restarts the bot (a prerestore snapshot is taken first). From 91ae820ab6569119c06c4d0e76fc83051851984b Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:01:47 +0100 Subject: [PATCH 05/21] test(tickets): cover the ticket access guard in the v2 tickets router requireTicketAccess was removed with the classic dashboard, and its two tests in classicBanner.test.js with it. The same guarantees are now tested against the current guard (ticketAccessLevel, run first by the v2 tickets router): a person without access gets a plain 403 with no page and no data, and a failed access lookup reaches the central error handler (a 500 without detail) instead of hanging the request. --- test/classicBanner.test.js | 68 +++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/test/classicBanner.test.js b/test/classicBanner.test.js index 32ead6c6..dd383035 100644 --- a/test/classicBanner.test.js +++ b/test/classicBanner.test.js @@ -2,16 +2,20 @@ // classic dashboard chrome: no htmx, Alpine, sidebar, retirement banner or inline script. They are // standalone pages in the new dashboard's look (/public.css, /theme.js). This file runs with login // enabled; test/publicPagesOpen.test.js covers open mode. -import { browser, setProfiles, signIn, startOAuthApp } from './helpers/oauthApp.js'; +import { D_GUILD, browser, setProfiles, signIn, startOAuthApp } from './helpers/oauthApp.js'; import { SAMPLES, capture, plainProblems, renderError, renderSample } from './helpers/publicViews.js'; import test from 'node:test'; import assert from 'node:assert/strict'; import { randomBytes } from 'node:crypto'; +import express from 'express'; await startOAuthApp(); const { pkceChallenge } = await import('../src/db/apiTokens.js'); const { createApp } = await import('../src/web/server.js'); const { forbidOwner } = await import('../src/web/middleware/auth.js'); +const { ticketAccessLevel } = await import('../src/web/middleware/ticketAccess.js'); +const { ticketsRouter } = await import('../src/web/routes/v2Staff.js'); +const { log } = await import('../src/lib/log.js'); const { LimitError } = await import('../src/lib/limits.js'); const { RoleGrantError } = await import('../src/web/middleware/roleGrants.js'); @@ -156,3 +160,65 @@ test('the error page renders with and without a title, heading and message', asy assertPlain(await renderError({}), 'bare error'); assertPlain(await renderError({ title: 'x', heading: 'y', message: 'z' }, { authEnabled: false }), 'error'); }); + +// The ticket access guard (ticketAccess.js) is the first thing the tickets router of the v2 API runs +// (v2Staff.js). This mounts that router behind the server's own central error handler, with a +// signed-in account chosen per request, to test what the guard answers. +async function withTicketsRouter(run) { + const central = app.router.stack.map((l) => l.handle).find((h) => h.length === 4); + const mini = express(); + mini.use((req, _res, next) => { + req.account = + req.get('x-lookup') === 'fails' + ? { + id: 'acc-1', + get identities() { + throw new Error('lookup failed: secret detail'); + }, + } + : { id: 'acc-1', identities: [] }; + next(); + }); + mini.use('/guilds/:guildId/tickets', ticketsRouter); + mini.use(central); + const server = await new Promise((resolve) => { + const srv = mini.listen(0, () => resolve(srv)); + }); + try { + return await run(`http://127.0.0.1:${server.address().port}/guilds/${D_GUILD}/tickets`); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +test('the ticket access guard refuses a person without access with a plain answer: no page, no data', async () => { + await withTicketsRouter(async (url) => { + for (const accept of ['text/html', 'application/json']) { + const res = await fetch(url, { headers: { accept } }); + assert.equal(res.status, 403); + assert.match(res.headers.get('content-type') ?? '', /application\/json/); + const body = await res.text(); + assert.deepEqual(JSON.parse(body), { + error: 'You need Manage Server, or a configured staff role, to view tickets here.', + }); + assert.doesNotMatch(body, /<|htmx|alpine|sidebar/i); + } + }); +}); + +test('a failed ticket access lookup goes to the error handler instead of hanging the request', async (t) => { + const logged = t.mock.method(log, 'error', () => {}); + await ticketAccessLevel({ account: { id: 'acc-1', identities: [] } }).then( + () => assert.fail('a lookup without a server must not turn into a level'), + (err) => assert.ok(err instanceof Error, 'the lookup rejects, it is not swallowed into a level') + ); + await withTicketsRouter(async (url) => { + const res = await fetch(url, { + headers: { accept: 'text/html', 'x-lookup': 'fails' }, + signal: AbortSignal.timeout(5000), + }); + assert.equal(res.status, 500, 'answered, not hung'); + assert.deepEqual(await res.json(), { error: 'Internal server error' }, 'no detail of the failure'); + }); + assert.equal(logged.mock.callCount(), 1, 'recorded once, with the route pattern'); +}); From f5e4916954c856713bd72f2e16b7cf94d5e2a46f Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:54:21 +0100 Subject: [PATCH 06/21] fix(live-alerts): decide item fault or request fault from the poll's own evidence The count rule (more than 10 rejected values means the request is at fault) let one guild with 11 bad values bring the outage back, and quarantined every good value on small instances when the platform answered 403 for a while. - A value rejected on its own is at fault when at least one request of the poll succeeded, however many there are. When no request succeeded the request is at fault: nothing is quarantined and the poll fails as before. - Only 400, 404 and 422 can mean a bad value; 403, 408, 409, 425 and the rest fail the poll. - The cost is capped by a per-poll request budget; batches not reached are left out of that poll, not failed, and each poll quarantines what it found. - The job loop leaves alone every value the poll did not resolve (quarantined, rejected, over budget) instead of asking the quarantine again later, which could end a live post by mistake. - One aggregated warning per poll, only for newly rejected values. - The user lookup of Twitch no longer fails the poll. --- src/modules/kickAlerts.js | 34 ++-- src/modules/lib/batchIsolation.js | 114 ++++++++---- src/modules/twitchAlerts.js | 55 +++--- test/liveAlertsIsolation.test.js | 286 ++++++++++++++++++++++++++---- 4 files changed, 389 insertions(+), 100 deletions(-) diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index 4ea0f540..f127ae83 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -17,7 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; -import { createQuarantine, fetchIsolated } from './lib/batchIsolation.js'; +import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -110,13 +110,17 @@ export const rejectedSlugs = createQuarantine(); /** * Live channels for a set of slugs, keyed by lowercased slug. Only currently-live - * channels are included. A slug that Kick rejects is isolated (see fetchIsolated), skipped - * and reported through `onRejected(slug, err, waitMs)`. + * channels are included. A slug that Kick rejects is isolated (see fetchIsolated) and left out. + * `report({ rejected, unresolved })` is called once with the slugs rejected now and every slug + * this poll knows nothing about. */ -export async function fetchLiveChannels(slugs, { onRejected } = {}) { +export async function fetchLiveChannels(slugs, { report } = {}) { const out = new Map(); - const found = await fetchIsolated(slugs, 50, getChannels, { quarantine: rejectedSlugs, onRejected }); - for (const ch of found) { + const { results, rejected, unresolved } = await fetchIsolated(slugs, 50, getChannels, { + quarantine: rejectedSlugs, + }); + report?.({ rejected, unresolved }); + for (const ch of results) { if (ch?.stream?.is_live && ch.slug) out.set(String(ch.slug).toLowerCase(), ch); } return out; @@ -188,14 +192,15 @@ export async function tick() { const slugs = [...new Set(jobs.map((j) => j.alert.slug))]; rejectedSlugs.prune(new Set(slugs)); let live; + let unresolved = new Set(); try { live = await fetchLiveChannels(slugs, { - onRejected: (slug, err, waitMs) => { - const guilds = [...new Set(jobs.filter((j) => j.alert.slug === slug).map((j) => j.guildId))]; - log.warn( - 'kick-alerts', - `Kick rejected slug "${slug}" (${err.message}); skipping it for ${Math.round(waitMs / 60_000)} min. Saved in: ${guilds.join(', ')}` - ); + report: ({ rejected, unresolved: left }) => { + unresolved = left; + const warning = rejectedWarning('Kick slugs', rejected, (slug) => [ + ...new Set(jobs.filter((j) => j.alert.slug === slug).map((j) => j.guildId)), + ]); + if (warning) log.warn('kick-alerts', warning); }, }); } catch (err) { @@ -204,8 +209,9 @@ export async function tick() { } for (const { guildId, alert } of jobs) { - // Rejected by Kick: neither live nor offline as far as we know, so leave its state alone. - if (rejectedSlugs.has(alert.slug)) continue; + // Not looked up this poll (rejected by Kick, waiting out a backoff, or over the request + // budget): neither live nor offline as far as we know, so leave its state alone. + if (unresolved.has(alert.slug)) continue; const channel = live.get(alert.slug); if (!channel) { const row = await seenRow(guildId, SCOPE, alert.slug); diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js index 9cb8ff60..dd14c67a 100644 --- a/src/modules/lib/batchIsolation.js +++ b/src/modules/lib/batchIsolation.js @@ -3,23 +3,39 @@ // The Twitch and Kick alert modules poll every server's streamers together. If the platform // answers a batch with a 4xx because of one stored login or slug, that single value must not stop // the poll for every other server. fetchIsolated() splits such a batch in half until the rejected -// item stands alone, quarantines that item with a growing backoff, and keeps the results of the -// rest. Other failures (5xx, network errors, 401, 429) still throw: they are not about one item -// and retrying the whole poll later is right. +// item stands alone, and keeps the results of the rest. After the poll the rejected items are +// quarantined with a growing backoff, so the next polls leave them out. +// +// Item fault or request fault? The poll's own evidence decides. If at least one request of the +// poll succeeded, an item that is rejected on its own is at fault, however many there are. If no +// request of the poll succeeded, the request is at fault (bad credentials, a blocked address, an +// outage): nothing is quarantined and the poll fails like any other failed poll. Only the +// statuses that can mean "this value" count (400, 404, 422); 5xx, network errors, 401, 429 and the +// other 4xx fail the poll. +// +// The cost is capped: a poll makes at most `batches + EXTRA_REQUESTS` requests. When that runs +// out, the batches not yet resolved are left out of this poll (not failed); each poll quarantines +// the values it found, so the next poll resolves more. -/** More rejected items than this in one poll means the request is at fault, not the items. */ -const MAX_REJECTED = 10; +/** Requests per poll beyond one per batch, for finding rejected items. */ +export const EXTRA_REQUESTS = 120; + +/** Statuses that can mean "this value was refused", as against a problem with the request. */ +const VALUE_STATUSES = new Set([400, 404, 422]); -/** Status codes that mean "the platform rejected the request itself", not "try again later". */ export function isRejectedStatus(status) { - return Number.isInteger(status) && status >= 400 && status < 500 && status !== 401 && status !== 429; + return VALUE_STATUSES.has(status); } /** * A set of items to leave out of polls for a while. Each repeated rejection doubles the wait. * @param {{ baseMs?: number, maxMs?: number, now?: () => number }} [o] */ -export function createQuarantine({ baseMs = 10 * 60_000, maxMs = 6 * 60 * 60_000, now = Date.now } = {}) { +export function createQuarantine({ + baseMs = 10 * 60_000, + maxMs = 6 * 60 * 60_000, + now = () => Date.now(), +} = {}) { /** @type {Map} */ const entries = new Map(); return { @@ -35,6 +51,10 @@ export function createQuarantine({ baseMs = 10 * 60_000, maxMs = 6 * 60 * 60_000 entries.set(key, { until: now() + wait, strikes }); return wait; }, + /** How many times the item was quarantined in a row (0 when never or cleared). */ + strikes(key) { + return entries.get(key)?.strikes ?? 0; + }, /** Forget an item that the platform accepted again. */ clear(key) { entries.delete(key); @@ -56,32 +76,34 @@ export function createQuarantine({ baseMs = 10 * 60_000, maxMs = 6 * 60 * 60_000 * @param {string[]} items * @param {number} size * @param {(batch: string[]) => Promise} fetchBatch throws an error with `.status` on a non-OK answer - * @param {{ quarantine: ReturnType, onRejected?: (item: string, err: Error, waitMs: number) => void, maxRejected?: number }} o - * @returns {Promise} the results of every accepted item + * @param {{ quarantine: ReturnType, maxRequests?: number }} o + * @returns {Promise<{ + * results: T[], + * rejected: Array<{ item: string, err: Error, waitMs: number, strikes: number }>, + * unresolved: Set, + * }>} `unresolved`: the items this poll knows nothing about (quarantined before, rejected now, or + * not reached when the request budget ran out); their stored state must be left alone. */ -export async function fetchIsolated( - items, - size, - fetchBatch, - { quarantine, onRejected, maxRejected = MAX_REJECTED } -) { - const out = []; - const added = []; +export async function fetchIsolated(items, size, fetchBatch, { quarantine, maxRequests }) { + const results = []; + const unresolved = new Set(); + const found = []; // rejected on their own: { item, err } + let requests = 0; + let succeeded = 0; + const budget = maxRequests ?? Math.ceil(items.length / size) + EXTRA_REQUESTS; + const run = async (batch) => { + if (requests >= budget) { + for (const item of batch) unresolved.add(item); + return; + } + requests += 1; try { - out.push(...(await fetchBatch(batch))); + results.push(...(await fetchBatch(batch))); } catch (err) { if (!isRejectedStatus(err?.status)) throw err; if (batch.length === 1) { - // Many different items rejected in one poll: the problem is the request (a bad app - // credential, say), not the items. Undo and fail the poll like any other error. - if (added.length >= maxRejected) { - for (const item of added) quarantine.clear(item); - throw err; - } - added.push(batch[0]); - const waitMs = quarantine.add(batch[0]); - onRejected?.(batch[0], err, waitMs); + found.push({ item: batch[0], err }); return; } const mid = Math.ceil(batch.length / 2); @@ -89,9 +111,41 @@ export async function fetchIsolated( await run(batch.slice(mid)); return; } + succeeded += 1; for (const item of batch) quarantine.clear(item); }; - const live = items.filter((item) => !quarantine.has(item)); + + const live = items.filter((item) => { + if (!quarantine.has(item)) return true; + unresolved.add(item); + return false; + }); for (let i = 0; i < live.length; i += size) await run(live.slice(i, i + size)); - return out; + + // Nothing succeeded: the request is at fault, not the items. Quarantine nothing. + if (found.length > 0 && succeeded === 0) throw found[0].err; + + const rejected = found.map(({ item, err }) => { + const waitMs = quarantine.add(item); + unresolved.add(item); + return { item, err, waitMs, strikes: quarantine.strikes(item) }; + }); + return { results, rejected, unresolved }; +} + +/** + * One log line for the values a poll just quarantined, naming the servers that saved them. Only + * first strikes are worth a warning (a value that comes back and is rejected again is not news); + * returns null when there is nothing new to say. + * @param {string} label e.g. 'Twitch logins' + * @param {Array<{ item: string, waitMs: number, strikes: number }>} rejected + * @param {(item: string) => string[]} savedIn the servers that stored the value + */ +export function rejectedWarning(label, rejected, savedIn) { + const fresh = rejected.filter((r) => r.strikes === 1); + if (fresh.length === 0) return null; + const shown = fresh.slice(0, 10).map((r) => `"${r.item}" (${savedIn(r.item).join(', ')})`); + const more = fresh.length > shown.length ? ` and ${fresh.length - shown.length} more` : ''; + const mins = Math.round(Math.min(...fresh.map((r) => r.waitMs)) / 60_000); + return `The platform rejected ${fresh.length} of the ${label} that servers saved: ${shown.join(', ')}${more}. Each is skipped for a while (first ${mins} min, longer each time it is rejected again), so the others keep working.`; } diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 5027b4ce..6fffeca6 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -17,7 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; -import { createQuarantine, fetchIsolated } from './lib/batchIsolation.js'; +import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -105,25 +105,36 @@ export const rejectedLogins = createQuarantine(); /** * Live streams for a set of logins, keyed by lowercased login. A login that Helix rejects is - * isolated (see fetchIsolated), skipped and reported through `onRejected(login, err, waitMs)`. + * isolated (see fetchIsolated) and left out. `report({ rejected, unresolved })` is called once with + * the logins rejected now and every login this poll knows nothing about. */ -export async function fetchLiveStreams(logins, { onRejected } = {}) { +export async function fetchLiveStreams(logins, { report } = {}) { const out = new Map(); - const found = await fetchIsolated(logins, 100, (batch) => helix('/streams', { user_login: batch }), { - quarantine: rejectedLogins, - onRejected, - }); - for (const s of found) out.set(s.user_login.toLowerCase(), s); + const { results, rejected, unresolved } = await fetchIsolated( + logins, + 100, + (batch) => helix('/streams', { user_login: batch }), + { quarantine: rejectedLogins } + ); + report?.({ rejected, unresolved }); + for (const s of results) out.set(s.user_login.toLowerCase(), s); return out; } -/** User records (for avatar / display name), keyed by lowercased login. A rejected login is just omitted. */ +/** + * User records (for avatar / display name), keyed by lowercased login. They only decorate the + * announcement, so a failure here (or a rejected login) leaves them out instead of failing the poll. + */ export async function fetchUsers(logins) { const out = new Map(); - const found = await fetchIsolated(logins, 100, (batch) => helix('/users', { login: batch }), { - quarantine: createQuarantine(), - }); - for (const u of found) out.set(u.login.toLowerCase(), u); + try { + const { results } = await fetchIsolated(logins, 100, (batch) => helix('/users', { login: batch }), { + quarantine: createQuarantine(), + }); + for (const u of results) out.set(u.login.toLowerCase(), u); + } catch (err) { + log.debug('twitch-alerts', 'user lookup failed:', err.message); + } return out; } @@ -191,14 +202,15 @@ export async function tick() { rejectedLogins.prune(new Set(logins)); let streams; let users; + let unresolved = new Set(); try { streams = await fetchLiveStreams(logins, { - onRejected: (login, err, waitMs) => { - const guilds = [...new Set(jobs.filter((j) => j.alert.login === login).map((j) => j.guildId))]; - log.warn( - 'twitch-alerts', - `Twitch rejected login "${login}" (${err.message}); skipping it for ${Math.round(waitMs / 60_000)} min. Saved in: ${guilds.join(', ')}` - ); + report: ({ rejected, unresolved: left }) => { + unresolved = left; + const warning = rejectedWarning('Twitch logins', rejected, (login) => [ + ...new Set(jobs.filter((j) => j.alert.login === login).map((j) => j.guildId)), + ]); + if (warning) log.warn('twitch-alerts', warning); }, }); users = await fetchUsers([...streams.keys()]); @@ -208,8 +220,9 @@ export async function tick() { } for (const { guildId, alert } of jobs) { - // Rejected by Twitch: neither live nor offline as far as we know, so leave its state alone. - if (rejectedLogins.has(alert.login)) continue; + // Not looked up this poll (rejected by Twitch, waiting out a backoff, or over the request + // budget): neither live nor offline as far as we know, so leave its state alone. + if (unresolved.has(alert.login)) continue; const stream = streams.get(alert.login); if (!stream) { const post = await announcedPost(guildId, alert.login); diff --git a/test/liveAlertsIsolation.test.js b/test/liveAlertsIsolation.test.js index e19d3183..dff3d732 100644 --- a/test/liveAlertsIsolation.test.js +++ b/test/liveAlertsIsolation.test.js @@ -7,8 +7,9 @@ await import('./helpers/tmpDb.js'); const { default: test } = await import('node:test'); const { default: assert } = await import('node:assert/strict'); const { runtime } = await import('../src/runtime.js'); +const { log } = await import('../src/lib/log.js'); const { setGuildModule } = await import('../src/db/modules.js'); -const { createQuarantine, fetchIsolated, isRejectedStatus } = +const { createQuarantine, fetchIsolated, isRejectedStatus, rejectedWarning } = await import('../src/modules/lib/batchIsolation.js'); const twitch = await import('../src/modules/twitchAlerts.js'); const kick = await import('../src/modules/kickAlerts.js'); @@ -19,20 +20,18 @@ const VICTIM = '900000000000000502'; const rejected = (status) => Object.assign(new Error(`rejected ${status}`), { status }); -test('isRejectedStatus: 4xx except 401/429 only', () => { - assert.equal(isRejectedStatus(400), true); - assert.equal(isRejectedStatus(404), true); - assert.equal(isRejectedStatus(401), false); - assert.equal(isRejectedStatus(429), false); - assert.equal(isRejectedStatus(500), false); - assert.equal(isRejectedStatus(undefined), false); +test('isRejectedStatus: only 400, 404 and 422 can mean "this value"', () => { + for (const status of [400, 404, 422]) assert.equal(isRejectedStatus(status), true, String(status)); + for (const status of [401, 403, 408, 409, 425, 429, 500, 503, undefined]) { + assert.equal(isRejectedStatus(status), false, String(status)); + } }); test('fetchIsolated: bisects to the rejected item and keeps the rest', async () => { const quarantine = createQuarantine(); const calls = []; const items = Array.from({ length: 8 }, (_, i) => `item${i}`); - const got = await fetchIsolated( + const out = await fetchIsolated( items, 8, async (batch) => { @@ -42,44 +41,113 @@ test('fetchIsolated: bisects to the rejected item and keeps the rest', async () }, { quarantine } ); - assert.deepEqual(got.sort(), items.filter((i) => i !== 'item5').sort()); + assert.deepEqual(out.results.sort(), items.filter((i) => i !== 'item5').sort()); + assert.deepEqual( + out.rejected.map((r) => [r.item, r.strikes]), + [['item5', 1]] + ); + assert.deepEqual([...out.unresolved], ['item5']); assert.equal(quarantine.has('item5'), true); assert.equal(quarantine.has('item4'), false); assert.ok(calls.length <= 1 + 2 * 3 + 1, `bounded request count, got ${calls.length}`); }); -test('fetchIsolated: a non-4xx failure still fails the whole poll', async () => { +test('fetchIsolated: any number of rejected items is fine while the rest succeeds', async () => { const quarantine = createQuarantine(); - await assert.rejects( - fetchIsolated(['a', 'b'], 2, async () => Promise.reject(rejected(503)), { quarantine }), - /rejected 503/ - ); - assert.equal(quarantine.size, 0); - await assert.rejects( - fetchIsolated(['a', 'b'], 2, async () => Promise.reject(rejected(429)), { quarantine }), - /rejected 429/ + const items = [...Array.from({ length: 15 }, (_, i) => `_bad${i}`), 'victim']; + const out = await fetchIsolated( + items, + 100, + async (batch) => { + if (batch.some((i) => i.startsWith('_'))) throw rejected(400); + return batch; + }, + { quarantine } ); - assert.equal(quarantine.size, 0); + assert.deepEqual(out.results, ['victim']); + assert.equal(out.rejected.length, 15); + assert.equal(quarantine.size, 15); }); -test('fetchIsolated: when everything is rejected the request is at fault, nothing is quarantined', async () => { +test('fetchIsolated: a failure that is not about a value still fails the whole poll', async () => { + for (const status of [503, 429, 401, 403, 408, 425]) { + const quarantine = createQuarantine(); + await assert.rejects( + fetchIsolated(['a', 'b'], 2, async () => Promise.reject(rejected(status)), { quarantine }), + new RegExp(`rejected ${status}`) + ); + assert.equal(quarantine.size, 0, String(status)); + } +}); + +test('fetchIsolated: when no request of the poll succeeded the request is at fault, nothing is quarantined', async () => { + for (const count of [3, 40]) { + const quarantine = createQuarantine(); + const items = Array.from({ length: count }, (_, i) => `item${i}`); + await assert.rejects( + fetchIsolated(items, 50, async () => Promise.reject(rejected(400)), { quarantine }), + /rejected 400/ + ); + assert.equal(quarantine.size, 0, `${count} items`); + } +}); + +test('fetchIsolated: a request budget leaves the rest of the poll out instead of failing it', async () => { const quarantine = createQuarantine(); - const items = Array.from({ length: 40 }, (_, i) => `item${i}`); + const items = [ + ...Array.from({ length: 8 }, (_, i) => `ok${i}`), + ...Array.from({ length: 8 }, (_, i) => `_bad${i}`), + ]; let calls = 0; - await assert.rejects( - fetchIsolated( - items, - 40, - async () => { - calls++; - throw rejected(403); - }, - { quarantine } - ), - /rejected 403/ + const out = await fetchIsolated( + items, + 4, + async (batch) => { + calls++; + if (batch.some((i) => i.startsWith('_'))) throw rejected(400); + return batch; + }, + { quarantine, maxRequests: 6 } + ); + assert.equal(calls, 6); + assert.ok(out.results.length > 0, 'what was fetched is kept'); + assert.ok(out.unresolved.size > 0, 'unreached items are reported as unresolved'); + for (const item of out.results) assert.ok(!out.unresolved.has(item)); +}); + +test('fetchIsolated: a value quarantined earlier is not asked about, and comes back after its time', async () => { + let now = 1_000; + const quarantine = createQuarantine({ baseMs: 100, now: () => now }); + quarantine.add('old'); + const asked = []; + const ask = async (batch) => { + asked.push(...batch); + return batch; + }; + const first = await fetchIsolated(['old', 'new'], 10, ask, { quarantine }); + assert.deepEqual(asked, ['new']); + assert.deepEqual([...first.unresolved], ['old']); + now += 101; + const second = await fetchIsolated(['old', 'new'], 10, ask, { quarantine }); + assert.deepEqual(second.results.sort(), ['new', 'old']); + assert.equal(quarantine.strikes('old'), 0, 'accepted again: the strikes reset'); +}); + +test('rejectedWarning: one line for the new values, nothing for a repeat', () => { + const r = (item, strikes) => ({ item, waitMs: 600_000, strikes }); + const line = rejectedWarning('Twitch logins', [r('a', 1), r('b', 1), r('c', 2)], (i) => [`guild-${i}`]); + assert.match(line, /rejected 2 of the Twitch logins/); + assert.match(line, /"a" \(guild-a\)/); + assert.doesNotMatch(line, /"c"/); + assert.equal( + rejectedWarning('Twitch logins', [r('c', 2)], () => []), + null + ); + const many = Array.from({ length: 30 }, (_, i) => r(`v${i}`, 1)); + assert.match( + rejectedWarning('Kick slugs', many, () => ['g']), + /and 20 more/ ); - assert.equal(quarantine.size, 0); - assert.ok(calls < 40, `gave up early, made ${calls} calls`); }); test('quarantine: backoff doubles, expires, and clears', () => { @@ -221,3 +289,151 @@ test('kick tick: a rejected slug does not stop alerts for another server', async runtime.client = null; } }); + +// --- tick-level: many poisoned values, request-level failures, recovery ------------------------- + +const twitchStream = (login) => ({ + id: `s-${login}`, + user_login: login, + user_name: login, + title: 'hi', + game_name: 'g', + viewer_count: 3, + started_at: '2026-11-16T00:00:00Z', + thumbnail_url: '', +}); + +test('twitch tick: 15 poisoned logins in one guild do not stop another guild, and each is quarantined', async (t) => { + const warned = t.mock.method(log, 'warn', () => {}); + const sent = []; + installClient(sent); + const poisoned = Array.from({ length: 15 }, (_, i) => ({ login: `_many${i}`, channelId: CH_A })); + await setGuildModule(ATTACKER, 'twitch-alerts', { enabled: true, config: { alerts: poisoned } }); + await setGuildModule(VICTIM, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: 'victim15', channelId: CH_A }] }, + }); + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.twitch.tv') return Response.json({ access_token: 't', expires_in: 3600 }); + const logins = u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login'); + if (logins.some((l) => l.startsWith('_'))) return new Response('{}', { status: 400 }); + if (u.pathname.endsWith('/streams')) { + return Response.json({ data: logins.filter((l) => l === 'victim15').map(twitchStream) }); + } + return Response.json({ data: [] }); + }; + try { + await twitch.tick(); + assert.equal(sent.length, 1); + assert.equal(sent[0].guild, VICTIM); + for (const p of poisoned) assert.equal(twitch.rejectedLogins.has(p.login), true, p.login); + assert.equal(warned.mock.callCount(), 1, 'one aggregated warning, not one per value'); + assert.match(String(warned.mock.calls[0].arguments[1]), /rejected 15 of the Twitch logins/); + // The next poll asks about none of them and warns about none. + await twitch.tick(); + assert.equal(warned.mock.callCount(), 1); + assert.equal(sent.length, 1); + } finally { + globalThis.fetch = realFetch; + } +}); + +test('twitch tick: a request-level 403 quarantines no good value and fails the poll like before', async (t) => { + const failed = t.mock.method(log, 'error', () => {}); + const sent = []; + installClient(sent); + await setGuildModule(ATTACKER, 'twitch-alerts', { enabled: false, config: { alerts: [] } }); + await setGuildModule(VICTIM, 'twitch-alerts', { + enabled: true, + config: { alerts: ['waf001', 'waf002', 'waf003'].map((login) => ({ login, channelId: CH_A })) }, + }); + const realFetch = globalThis.fetch; + let blocked = true; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.twitch.tv') return Response.json({ access_token: 't', expires_in: 3600 }); + if (blocked) return new Response('{}', { status: 403 }); + const logins = u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login'); + if (u.pathname.endsWith('/streams')) return Response.json({ data: logins.map(twitchStream) }); + return Response.json({ data: [] }); + }; + try { + await twitch.tick(); + assert.equal(sent.length, 0); + assert.equal(failed.mock.callCount(), 1, 'poll failed, as before'); + for (const login of ['waf001', 'waf002', 'waf003']) assert.equal(twitch.rejectedLogins.has(login), false); + // The outage ends: the very next poll announces them all. + blocked = false; + await twitch.tick(); + assert.equal(sent.length, 3); + } finally { + globalThis.fetch = realFetch; + } +}); + +test('twitch tick: a quarantined login that Twitch accepts again is announced and its strikes reset', async (t) => { + t.mock.method(log, 'warn', () => {}); + t.mock.timers.enable({ apis: ['Date'], now: Date.now() }); + const sent = []; + installClient(sent); + await setGuildModule(ATTACKER, 'twitch-alerts', { enabled: false, config: { alerts: [] } }); + await setGuildModule(VICTIM, 'twitch-alerts', { + enabled: true, + config: { + alerts: [ + { login: '_flaky1', channelId: CH_A }, + { login: 'steady1', channelId: CH_A }, + ], + }, + }); + const realFetch = globalThis.fetch; + let rejectFlaky = true; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.twitch.tv') return Response.json({ access_token: 't', expires_in: 3600 }); + const logins = u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login'); + if (rejectFlaky && logins.includes('_flaky1')) return new Response('{}', { status: 400 }); + if (u.pathname.endsWith('/streams')) return Response.json({ data: logins.map(twitchStream) }); + return Response.json({ data: [] }); + }; + try { + await twitch.tick(); + assert.equal(sent.length, 1, 'steady1 announced, _flaky1 not'); + assert.equal(twitch.rejectedLogins.strikes('_flaky1'), 1); + rejectFlaky = false; + t.mock.timers.tick(11 * 60_000); // past the first backoff + await twitch.tick(); + assert.equal(sent.length, 2, '_flaky1 announced once Twitch accepts it'); + assert.equal(twitch.rejectedLogins.strikes('_flaky1'), 0); + } finally { + globalThis.fetch = realFetch; + } +}); + +test('kick tick: a request-level 403 quarantines no good slug', async (t) => { + const failed = t.mock.method(log, 'error', () => {}); + const sent = []; + installClient(sent); + await setGuildModule(ATTACKER, 'kick-alerts', { enabled: false, config: { alerts: [] } }); + await setGuildModule(VICTIM, 'kick-alerts', { + enabled: true, + config: { alerts: ['wafk1', 'wafk2'].map((slug) => ({ slug, channelId: CH_A })) }, + }); + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.kick.com') return Response.json({ access_token: 't', expires_in: 3600 }); + return new Response('{}', { status: 403 }); + }; + try { + await kick.tick(); + assert.equal(sent.length, 0); + assert.equal(failed.mock.callCount(), 1); + assert.equal(kick.rejectedSlugs.has('wafk1') || kick.rejectedSlugs.has('wafk2'), false); + } finally { + globalThis.fetch = realFetch; + runtime.client = null; + } +}); From e4e2776df74f352a172836c443c80cbb053e6946 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:54:24 +0100 Subject: [PATCH 07/21] fix(tickets): route a DM by open tickets, never by a cut-short lookup - The servers where the member has an open ticket come from the database (new openTicketGuildKeys, indexed on Postgres) and are checked first, outside the caps, so a follow-up reaches its ticket after a restart, with the intent off and in large servers. - A lookup that the caps cut short never opens a new ticket: the member is asked to send again. Only the single open ticket (a complete list) is routed to. - Per-user budget (16 fetches a minute) next to the bot's 120; open-ticket checks and the server pick are not counted against the shared budget. - An incomplete answer is not reused, so a retry makes progress; servers are asked in the order not asked yet / longest ago, and the not-a-member answer is remembered for 10 minutes, so the first minute's answers do not expire before the last servers are reached. - A server that refuses member lookups (403) is skipped for that time instead of keeping the member waiting. - A refused server pick keeps the stashed message and the menu. --- docs/modules/tickets.md | 16 +- src/bot/events/dmTickets.js | 54 +++++-- src/db/tickets.js | 7 + src/modules/tickets.js | 148 ++++++++++++----- test/ticketGuildLookup.test.js | 288 ++++++++++++++++++++++++++------- 5 files changed, 392 insertions(+), 121 deletions(-) diff --git a/docs/modules/tickets.md b/docs/modules/tickets.md index e0df6f95..4a83ac90 100644 --- a/docs/modules/tickets.md +++ b/docs/modules/tickets.md @@ -33,13 +33,15 @@ server. Staff reply as **Staff** from the ticket page; the member sees Sylo's DM **Close ticket** sends a final reply (or the default closing notice if the box is empty) and closes it. The member can open a new one by DMing again. -To find the servers a member shares with the bot, Sylo uses its member cache first. A server -whose full member list is cached (Server Members intent on) answers without any request to -Discord. For the others it asks Discord for the one member, at most 8 servers per message, at most -120 requests a minute for the whole bot, and it remembers "not a member" for two minutes. Without -the Server Members intent a member is still found this way; if a message could not check every -server yet, the member is asked to send it again a few seconds later, which continues with the -servers not checked yet. +To find the servers a member shares with the bot, Sylo first looks at the servers where the member +has an open ticket (from its own database, so this survives a restart): a follow-up message always +goes to the open ticket. Then it uses its member cache: a server whose full member list is cached +(Server Members intent on) answers without any request to Discord. For the other servers it asks +Discord for the one member: at most 8 servers per message, 16 requests a minute per member and 120 a +minute for the whole bot, and it remembers "not a member" (or a refused lookup) for ten minutes. +Without the Server Members intent a member is still found this way. If a message could not check +every server yet, Sylo never opens a new ticket from it: it asks the member to send the message again +a few seconds later, which continues with the servers not checked yet. `/forget` removes the member's ticket history: every message in their tickets, staff replies included. Replies already delivered by DM are not clawed back. diff --git a/src/bot/events/dmTickets.js b/src/bot/events/dmTickets.js index a9107a80..b664ef0a 100644 --- a/src/bot/events/dmTickets.js +++ b/src/bot/events/dmTickets.js @@ -6,7 +6,6 @@ import { guildKey, guildByKey, platformOfClient } from '../../platform/platformO import { config } from '../../config.js'; import { Events, StringSelectMenuBuilder, ActionRowBuilder } from 'discord.js'; import { lookupTicketGuilds, canOpenTicketIn, ingestUserDM } from '../../modules/tickets.js'; -import { getOpenTicket } from '../../db/tickets.js'; import { registerComponent } from '../lib/components.js'; import { handleQuestionnaireMessage, looksLikeQuestionnairePrompt } from '../lib/dmQuestionnaire.js'; import { lateAnswerWindowMs } from '../../modules/applications.js'; @@ -70,25 +69,31 @@ async function handleDM(message) { if (!payload.content && payload.attachments.length === 0) return; // Only the servers on the platform this DM arrived on: a user id means nothing on the other one. - const { guilds, incomplete } = await lookupTicketGuilds(message.author, message.client); - if (guilds.length === 0) { - // `incomplete`: the lookup is capped per message, so some servers were not checked yet. + const { guilds, openGuilds, openComplete, incomplete } = await lookupTicketGuilds( + message.author, + message.client + ); + + // The open tickets come from the database, so they are certain. A follow-up goes to the one + // server where the member has an open ticket, however much of the rest could be checked. + if (openComplete && openGuilds.length === 1) { + await ingestUserDM(openGuilds[0], message.author, payload); + await message.react('✅').catch(() => {}); + return; + } + // Some servers could not be checked yet (the lookup is capped per message). Never open a new + // ticket from that: the right server may be one of those. Ask for the message again; the + // retry continues with the servers not checked yet. + if (incomplete && openGuilds.length === 0) { await message - .reply( - incomplete - ? 'I could not check all servers yet. Please send your message again in a few seconds.' - : "I'm not set up to take messages for any server you're in right now." - ) + .reply('I could not check all servers yet. Please send your message again in a few seconds.') .catch(() => {}); return; } - - // If there's exactly one open ticket already, keep the conversation there. - const openFlags = await Promise.all(guilds.map((g) => getOpenTicket(guildKey(g), message.author.id))); - const openGuilds = guilds.filter((g, i) => openFlags[i]); - if (openGuilds.length === 1) { - await ingestUserDM(openGuilds[0], message.author, payload); - await message.react('✅').catch(() => {}); + if (guilds.length === 0) { + await message + .reply("I'm not set up to take messages for any server you're in right now.") + .catch(() => {}); return; } if (openGuilds.length === 0 && guilds.length === 1) { @@ -117,10 +122,25 @@ async function handleDM(message) { async function handleSelect(interaction) { const picked = guildByKey(interaction.values[0]); + const stashed = pending.get(interaction.user.id); + if (!stashed || Date.now() - stashed.at > PENDING_TTL) { + pending.delete(interaction.user.id); + await interaction + .update({ content: 'That request expired — send your message again.', components: [] }) + .catch(() => {}); + return; + } // Still one the member may open a ticket in (a member there, tickets on): the menu could be old. const guild = picked && (await canOpenTicketIn(interaction.user, picked)) ? picked : null; + if (!guild) { + // The message stays stashed and the menu stays up, so the member can pick again. + await interaction + .reply({ content: 'I could not confirm that server right now. Pick it again in a moment.' }) + .catch(() => {}); + return; + } const payload = takeStash(interaction.user.id); - if (!guild || !payload) { + if (!payload) { await interaction .update({ content: 'That request expired — send your message again.', components: [] }) .catch(() => {}); diff --git a/src/db/tickets.js b/src/db/tickets.js index 85a618c5..43c6f490 100644 --- a/src/db/tickets.js +++ b/src/db/tickets.js @@ -15,6 +15,7 @@ registerPostgresBootstrap(` ); CREATE INDEX IF NOT EXISTS idx_tickets_guild_status ON tickets (guild_id, status, last_at DESC); CREATE UNIQUE INDEX IF NOT EXISTS idx_tickets_open_user ON tickets (guild_id, user_id) WHERE status = 'open'; + CREATE INDEX IF NOT EXISTS idx_tickets_user_open ON tickets (user_id) WHERE status = 'open'; CREATE TABLE IF NOT EXISTS ticket_messages ( id SERIAL PRIMARY KEY, @@ -33,6 +34,7 @@ const openByUserStmt = prepare( "SELECT * FROM tickets WHERE guild_id = ? AND user_id = ? AND status = 'open'" ); const getStmt = prepare('SELECT * FROM tickets WHERE id = ?'); +const openGuildsStmt = prepare("SELECT guild_id FROM tickets WHERE user_id = ? AND status = 'open'"); const createStmt = prepare( ` INSERT INTO tickets (guild_id, user_id, status, created_at, last_at) @@ -88,6 +90,11 @@ export async function getOpenTicket(guildId, userId) { return toTicket(await openByUserStmt.get(guildId, userId)); } +/** The servers where the user has an open ticket (a DM to the bot continues one of them). */ +export async function openTicketGuildKeys(userId) { + return (await openGuildsStmt.all(userId)).map((r) => r.guild_id); +} + export async function getTicket(id) { return toTicket(await getStmt.get(id)); } diff --git a/src/modules/tickets.js b/src/modules/tickets.js index afc268bd..97198133 100644 --- a/src/modules/tickets.js +++ b/src/modules/tickets.js @@ -9,6 +9,7 @@ import { isModuleEnabled, getGuildModule } from '../db/modules.js'; import { notifyTicketActivity } from '../lib/push.js'; import { sweepMap } from '../lib/sweepMap.js'; import { + openTicketGuildKeys, openOrCreateTicket, getTicket, addTicketMessage, @@ -155,41 +156,56 @@ ${bubbles} // --- Which ticket servers is a DM sender in? -------------------------------------------------- // Any account can DM the bot, so the lookup must cost little however many servers the instance -// has. Order of work: (1) members already in the cache cost nothing; (2) when the member cache of -// a server is known to be complete (GuildMembers intent on and every member cached) a miss is a -// definite "not a member"; (3) only the rest need a REST member fetch, and those are capped per -// DM, per minute for the whole bot, remembered when the member does not exist, and tried first -// where the user was found before. Without the GuildMembers intent the member caches are sparse, -// so step 3 is the only way to find a real sender: it still works, a few servers per DM, and a -// sender whose servers were not all checked is told to send the message again, which continues -// with the servers not yet checked. +// has. Order of work: +// 1. The servers where the user has an open ticket come from the database (they survive a +// restart) and are checked first. They are not counted against the caps below: a user has few, +// and a follow-up must reach the server it belongs to. +// 2. Members already in the cache cost nothing. When the member cache of a server is known to be +// complete (GuildMembers intent on and every member cached) a miss means "not a member". +// 3. Only the rest need a REST member fetch. Those are capped per DM, per user per minute and per +// minute for the whole bot, remembered when the member does not exist (or the fetch is +// refused), and tried first where the user was found before, then where asked longest ago. +// Without the GuildMembers intent the member caches are sparse, so step 3 is the only way to find +// a member who has no open ticket: it still works, a few servers per message. A lookup that the +// caps cut short is `incomplete`; the DM handler then never opens a new ticket from it, it asks the +// member to send again, and the retry continues with the servers not yet checked. /** Most REST member fetches one DM may cause. */ export const MEMBER_PROBES_PER_DM = 8; -/** Most REST member fetches all DMs together may cause per minute. */ +/** Most REST member fetches one user may cause per minute. */ +export const MEMBER_PROBES_PER_USER_PER_MINUTE = 16; +/** Most REST member fetches all users together may cause per minute (open tickets are not counted). */ export const MEMBER_PROBES_PER_MINUTE = 120; -/** A "not a member" answer is remembered this long. */ -export const NOT_MEMBER_TTL_MS = 2 * 60_000; -/** The answer for one sender is reused this long (a per-user cooldown on the lookup). */ +/** A "not a member" answer, or a refused fetch, is remembered this long. */ +export const NOT_MEMBER_TTL_MS = 10 * 60_000; +/** A complete answer for one sender is reused this long (a per-user cooldown on the lookup). */ export const LOOKUP_REUSE_MS = 10_000; const HINT_TTL_MS = 24 * 60 * 60_000; /** `${userId}|${guildKey}` -> expiry of a "not a member" answer. @type {Map} */ const notMember = new Map(); +/** `${userId}|${guildKey}` -> when the user was last looked up there, so that unchecked servers go first. */ +const checkedAt = new Map(); /** userId -> servers the user was found in lately; only decides the order of the fetches. */ const foundBefore = new Map(); /** `${platform}|${userId}` -> { at, result } (result is a promise while the lookup runs). */ const lookups = new Map(); +/** userId -> { start, count }: the user's fetches in the current minute. */ +const userWindows = new Map(); let probeWindow = { start: 0, count: 0 }; sweepMap(notMember, (until, now) => until <= now); +sweepMap(checkedAt, (at, now) => now - at > HINT_TTL_MS); sweepMap(foundBefore, (h, now) => now - h.at > HINT_TTL_MS); sweepMap(lookups, (l, now) => now - l.at > LOOKUP_REUSE_MS); +sweepMap(userWindows, (w, now) => now - w.start >= 60_000, 60_000); -/** Test hook: forget every remembered answer and the probe budget. */ +/** Test hook: forget every remembered answer and the probe budgets. */ export function _resetTicketLookup() { notMember.clear(); foundBefore.clear(); + checkedAt.clear(); lookups.clear(); + userWindows.clear(); probeWindow = { start: 0, count: 0 }; } @@ -204,54 +220,97 @@ function memberCacheComplete(guild) { return Number.isFinite(guild.memberCount) && (guild.members.cache?.size ?? 0) >= guild.memberCount; } -/** Take one REST member fetch from the per-minute budget; false when it is used up. */ -function takeProbe() { +/** Take one REST member fetch from the user's and the bot's per-minute budgets; false when either is used up. */ +function takeProbe(userId) { const now = Date.now(); if (now - probeWindow.start >= 60_000) probeWindow = { start: now, count: 0 }; - if (probeWindow.count >= MEMBER_PROBES_PER_MINUTE) return false; + let w = userWindows.get(userId); + if (!w || now - w.start >= 60_000) { + w = { start: now, count: 0 }; + userWindows.set(userId, w); + } + if (probeWindow.count >= MEMBER_PROBES_PER_MINUTE || w.count >= MEMBER_PROBES_PER_USER_PER_MINUTE) { + return false; + } probeWindow.count++; + w.count++; return true; } -/** One REST member fetch. @returns {Promise} true member, false not a member, null unknown */ +/** + * One REST member fetch. + * @returns {Promise} true member; false not a member, or the platform refuses to + * say (remembered for NOT_MEMBER_TTL_MS, so it is not asked again at once); null unknown (a rate + * limit or a network error: not remembered) + */ async function probeMember(guild, userId) { + const remember = () => notMember.set(`${userId}|${guildKey(guild)}`, Date.now() + NOT_MEMBER_TTL_MS); + checkedAt.set(`${userId}|${guildKey(guild)}`, Date.now()); try { // A platform may answer "no such member" with null instead of an error. if (await guild.members.fetch(userId)) return true; - notMember.set(`${userId}|${guildKey(guild)}`, Date.now() + NOT_MEMBER_TTL_MS); + remember(); return false; } catch (err) { - if (err?.code === 10007 || err?.code === 10013 || err?.status === 404) { - notMember.set(`${userId}|${guildKey(guild)}`, Date.now() + NOT_MEMBER_TTL_MS); + // 10007 / 10013: unknown member / user. 403 / 50001: the bot may not look in this server. + if ([10007, 10013, 50001].includes(err?.code) || [403, 404].includes(err?.status)) { + remember(); return false; } - return null; // a rate limit or a network error says nothing about membership + return null; } } async function findTicketGuilds(user, client) { const found = []; + const openGuilds = []; + const byKey = new Map(); + for (const guild of client ? [...client.guilds.cache.values()] : allGuilds()) + byKey.set(guildKey(guild), guild); + + // 1. Open tickets, from the database: complete, and kept across restarts. + let openComplete = true; + const handled = new Set(); + for (const key of await openTicketGuildKeys(user.id)) { + const guild = byKey.get(key); + if (!guild) continue; // another platform, or Sylo left the server + handled.add(key); + if (!(await isModuleEnabled(key, 'tickets'))) continue; + const isMember = cachedMember(guild, user.id) ? true : await probeMember(guild, user.id); + if (isMember) { + found.push(guild); + openGuilds.push(guild); + } else if (isMember === null) openComplete = false; + } + + // 2. The cache. const toProbe = []; const now = Date.now(); - for (const guild of client ? [...client.guilds.cache.values()] : allGuilds()) { + for (const [key, guild] of byKey) { + if (handled.has(key)) continue; if (cachedMember(guild, user.id)) { - if (await isModuleEnabled(guildKey(guild), 'tickets')) found.push(guild); + if (await isModuleEnabled(key, 'tickets')) found.push(guild); continue; } if (memberCacheComplete(guild)) continue; - if ((notMember.get(`${user.id}|${guildKey(guild)}`) ?? 0) > now) continue; + if ((notMember.get(`${user.id}|${key}`) ?? 0) > now) continue; toProbe.push(guild); } - // Servers where this user was found before go first. + // 3. REST fetches, servers where this user was found before first. + // Then the ones not asked about yet, the longest ago first: with more servers than one minute's + // budget covers, the answers of the first minute may have expired before the last are reached. const hint = foundBefore.get(user.id)?.guilds; - if (hint) toProbe.sort((a, b) => Number(hint.has(guildKey(b))) - Number(hint.has(guildKey(a)))); - + const asked = (g) => checkedAt.get(`${user.id}|${guildKey(g)}`) ?? 0; + toProbe.sort( + (a, b) => + Number(hint?.has(guildKey(b)) ?? false) - Number(hint?.has(guildKey(a)) ?? false) || asked(a) - asked(b) + ); let probes = 0; - let incomplete = false; + let incomplete = !openComplete; for (const guild of toProbe) { if (!(await isModuleEnabled(guildKey(guild), 'tickets'))) continue; - if (probes >= MEMBER_PROBES_PER_DM || !takeProbe()) { + if (probes >= MEMBER_PROBES_PER_DM || !takeProbe(user.id)) { incomplete = true; break; } @@ -264,29 +323,40 @@ async function findTicketGuilds(user, client) { if (found.length) { foundBefore.set(user.id, { guilds: new Set(found.map(guildKey)), at: Date.now() }); } - return { guilds: found, incomplete }; + return { guilds: found, openGuilds, openComplete, incomplete }; } /** - * Guilds where the tickets module is enabled and the user is a member, and whether every - * candidate server was checked (`incomplete` when the fetch caps cut the lookup short). Pass the - * client the DM arrived on to look only at that platform's servers. The answer for one user is - * reused for a few seconds, so a burst of DMs does the work once. - * @returns {Promise<{ guilds: import('discord.js').Guild[], incomplete: boolean }>} + * Where can this sender open or continue a ticket? `guilds`: servers with tickets on that the user + * is a member of; `openGuilds`: those where the user has an open ticket, and `openComplete`: that + * list is certain; `incomplete`: some servers could not be checked yet (the caps), so `guilds` may + * be missing some. Pass the client the DM arrived on to look only at that platform's servers. + * A complete answer for one user is reused for a few seconds, so a burst of DMs does the work + * once; an incomplete one is not reused, so a retry makes progress. + * @returns {Promise<{ guilds: import('discord.js').Guild[], openGuilds: import('discord.js').Guild[], openComplete: boolean, incomplete: boolean }>} */ export function lookupTicketGuilds(user, client) { const key = `${client ? platformOfClient(client) : 'all'}|${user.id}`; const recent = lookups.get(key); if (recent && Date.now() - recent.at < LOOKUP_REUSE_MS) return recent.result; const result = findTicketGuilds(user, client); - lookups.set(key, { at: Date.now(), result }); - result.catch(() => lookups.delete(key)); + const entry = { at: Date.now(), result }; + lookups.set(key, entry); + result.then( + (r) => { + if (r.incomplete && lookups.get(key) === entry) lookups.delete(key); + }, + () => { + if (lookups.get(key) === entry) lookups.delete(key); + } + ); return result; } /** * Can this user open a ticket in this one server (tickets on, user a member)? Cheap: the member - * cache, or the DM lookup that just ran, or at most one fetch from the shared budget. + * cache, or the DM lookup that just ran, or one fetch. The server was one the bot offered in a + * menu a moment ago, so that one fetch is not counted against the budgets. */ export async function canOpenTicketIn(user, guild) { if (!(await isModuleEnabled(guildKey(guild), 'tickets'))) return false; @@ -296,7 +366,7 @@ export async function canOpenTicketIn(user, guild) { const { guilds } = await recent.result.catch(() => ({ guilds: [] })); if (guilds.some((g) => guildKey(g) === guildKey(guild))) return true; } - if (memberCacheComplete(guild) || !takeProbe()) return false; + if (memberCacheComplete(guild)) return false; return (await probeMember(guild, user.id)) === true; } diff --git a/test/ticketGuildLookup.test.js b/test/ticketGuildLookup.test.js index da922bb8..31b25fd5 100644 --- a/test/ticketGuildLookup.test.js +++ b/test/ticketGuildLookup.test.js @@ -1,35 +1,45 @@ -// A DM to the bot must not cost a member fetch in every ticket server of the instance. +// A DM to the bot must not cost a member fetch in every ticket server of the instance, and a +// lookup that the caps cut short must never send a DM to the wrong server. import './helpers/tmpDb.js'; import test from 'node:test'; import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import { setGuildModule } from '../src/db/modules.js'; +import { createTicket, getOpenTicket, ticketMessages, openTicketGuildKeys } from '../src/db/tickets.js'; import { lookupTicketGuilds, canOpenTicketIn, _resetTicketLookup, MEMBER_PROBES_PER_DM, MEMBER_PROBES_PER_MINUTE, + MEMBER_PROBES_PER_USER_PER_MINUTE, LOOKUP_REUSE_MS, + NOT_MEMBER_TTL_MS, } from '../src/modules/tickets.js'; import { register } from '../src/bot/events/dmTickets.js'; +import { routeComponent } from '../src/bot/lib/components.js'; +import { runtime } from '../src/runtime.js'; const SENDER = { id: '700000000000000001' }; let nextId = 800000000000000000n; /** * A ticket-enabled fake guild. `complete`: the member cache holds every member (so a miss means - * "not a member"). `members`: ids that really are members (a REST fetch finds them). + * "not a member"). `members`: ids that really are members (a REST fetch finds them). `refuse`: + * every fetch answers 403 (the bot may not look in this server). */ -async function ticketGuild(fetches, { complete = false, members = [], cached = [] } = {}) { +async function ticketGuild(fetches, { complete = false, members = [], cached = [], refuse = false } = {}) { const guild = { id: String(nextId++), + name: 'Test server', client: {}, memberCount: complete ? 3 : 5000, + channels: { cache: new Map(), fetch: async () => null }, members: { cache: new Map(cached.map((id) => [id, { id, partial: false }])), fetch: async (id) => { fetches.push({ guild: guild.id, id }); + if (refuse) throw Object.assign(new Error('Missing Access'), { status: 403, code: 50001 }); if (members.includes(id)) return { id }; throw Object.assign(new Error('Unknown Member'), { code: 10007 }); }, @@ -41,6 +51,7 @@ async function ticketGuild(fetches, { complete = false, members = [], cached = [ } const clientOf = (guilds) => ({ guilds: { cache: new Map(guilds.map((g) => [g.id, g])) } }); +const ids = (guilds) => guilds.map((g) => g.id); test('a sender in no server: complete caches answer without any REST fetch', async () => { _resetTicketLookup(); @@ -48,7 +59,7 @@ test('a sender in no server: complete caches answer without any REST fetch', asy const guilds = []; for (let i = 0; i < 30; i++) guilds.push(await ticketGuild(fetches, { complete: true })); const r = await lookupTicketGuilds(SENDER, clientOf(guilds)); - assert.deepEqual(r, { guilds: [], incomplete: false }); + assert.deepEqual([r.guilds, r.openGuilds, r.incomplete], [[], [], false]); assert.equal(fetches.length, 0); }); @@ -59,10 +70,7 @@ test('a cached member is found without a fetch, in the right server only', async const guilds = [a]; for (let i = 0; i < 20; i++) guilds.push(await ticketGuild(fetches, { complete: true })); const r = await lookupTicketGuilds(SENDER, clientOf(guilds)); - assert.deepEqual( - r.guilds.map((g) => g.id), - [a.id] - ); + assert.deepEqual(ids(r.guilds), [a.id]); assert.equal(fetches.length, 0); }); @@ -74,31 +82,50 @@ test('a cached member of a server with tickets off is not offered', async () => assert.deepEqual((await lookupTicketGuilds(SENDER, clientOf([g]))).guilds, []); }); -test('incomplete caches: fetches per DM are capped, remembered, and continue on the next DM', async (t) => { +test('incomplete caches: fetches are capped per DM and per user, remembered, and a retry makes progress', async (t) => { _resetTicketLookup(); t.mock.timers.enable({ apis: ['Date'], now: 1_000_000 }); const fetches = []; const guilds = []; - for (let i = 0; i < 20; i++) guilds.push(await ticketGuild(fetches)); + for (let i = 0; i < 40; i++) guilds.push(await ticketGuild(fetches)); const client = clientOf(guilds); const first = await lookupTicketGuilds(SENDER, client); assert.equal(fetches.length, MEMBER_PROBES_PER_DM); - assert.deepEqual(first, { guilds: [], incomplete: true }); + assert.equal(first.incomplete, true); - // Sending again at once reuses the answer: no new fetches (per-user cooldown). + // A retry at once is not answered from the first, incomplete answer: it checks the next servers. await lookupTicketGuilds(SENDER, client); - assert.equal(fetches.length, MEMBER_PROBES_PER_DM); + assert.equal(fetches.length, MEMBER_PROBES_PER_DM * 2); + assert.equal(new Set(fetches.map((f) => f.guild)).size, fetches.length, 'no server asked twice'); - // Later, the next servers are checked; the ones already known as "not a member" are skipped. - t.mock.timers.tick(LOOKUP_REUSE_MS + 1); - await lookupTicketGuilds(SENDER, client); + // The user's own budget for the minute is used up: no more fetches until it renews. + assert.equal(MEMBER_PROBES_PER_USER_PER_MINUTE, MEMBER_PROBES_PER_DM * 2); + const third = await lookupTicketGuilds(SENDER, client); assert.equal(fetches.length, MEMBER_PROBES_PER_DM * 2); + assert.equal(third.incomplete, true); + + t.mock.timers.tick(61_000); + for (let n = 0; n < 6 && (await lookupTicketGuilds(SENDER, client)).incomplete; n++) { + t.mock.timers.tick(61_000); + } + assert.equal(fetches.length, 40, 'each server was fetched exactly once'); +}); + +test('a complete answer is reused for a few seconds', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 1_500_000 }); + const fetches = []; + const guilds = [await ticketGuild(fetches), await ticketGuild(fetches)]; + await lookupTicketGuilds(SENDER, clientOf(guilds)); + await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.equal(fetches.length, 2); t.mock.timers.tick(LOOKUP_REUSE_MS + 1); - const last = await lookupTicketGuilds(SENDER, client); - assert.equal(fetches.length, 20, 'each server was fetched exactly once'); - assert.equal(last.incomplete, false); - assert.equal(new Set(fetches.map((f) => f.guild)).size, 20); + await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.equal(fetches.length, 2, 'the not-a-member answers are still remembered'); + t.mock.timers.tick(NOT_MEMBER_TTL_MS); + await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.equal(fetches.length, 4); }); test('a real member in a late server is found, and found first the next time', async (t) => { @@ -111,34 +138,29 @@ test('a real member in a late server is found, and found first the next time', a guilds.push(mine); const client = clientOf(guilds); - await lookupTicketGuilds(SENDER, client); // first 8 servers, not found - t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + const first = await lookupTicketGuilds(SENDER, client); // first 8 servers, not found + assert.equal(first.incomplete, true); const r = await lookupTicketGuilds(SENDER, client); - assert.deepEqual( - r.guilds.map((g) => g.id), - [mine.id] - ); + assert.deepEqual(ids(r.guilds), [mine.id]); - // Hint: the next lookup (everything forgotten except the hint) tries that server first. + // Hint: later (everything else forgotten) the next lookup tries that server first. t.mock.timers.tick(LOOKUP_REUSE_MS + 3 * 60_000); fetches.length = 0; const again = await lookupTicketGuilds(SENDER, client); assert.equal(fetches[0].guild, mine.id); - assert.deepEqual( - again.guilds.map((g) => g.id), - [mine.id] - ); + assert.deepEqual(ids(again.guilds), [mine.id]); }); -test('all DMs together stay within the per-minute fetch budget', async (t) => { +test('all users together stay within the per-minute fetch budget', async (t) => { _resetTicketLookup(); t.mock.timers.enable({ apis: ['Date'], now: 3_000_000 }); const fetches = []; const guilds = []; for (let i = 0; i < 20; i++) guilds.push(await ticketGuild(fetches)); const client = clientOf(guilds); - for (let u = 0; u < 40; u++) + for (let u = 0; u < 40; u++) { await lookupTicketGuilds({ id: `71000000000000${String(u).padStart(4, '0')}` }, client); + } assert.equal(fetches.length, MEMBER_PROBES_PER_MINUTE); const late = await lookupTicketGuilds({ id: '720000000000000000' }, client); assert.equal(late.incomplete, true); @@ -147,6 +169,20 @@ test('all DMs together stay within the per-minute fetch budget', async (t) => { assert.ok(fetches.length > MEMBER_PROBES_PER_MINUTE, 'the budget resets after a minute'); }); +test('one user cannot use the whole budget: another user still gets fetches', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 3_500_000 }); + const fetches = []; + const guilds = []; + for (let i = 0; i < 60; i++) guilds.push(await ticketGuild(fetches)); + const client = clientOf(guilds); + for (let n = 0; n < 20; n++) await lookupTicketGuilds(SENDER, client); // way over its own budget + assert.equal(fetches.filter((f) => f.id === SENDER.id).length, MEMBER_PROBES_PER_USER_PER_MINUTE); + const other = { id: '730000000000000009' }; + await lookupTicketGuilds(other, client); + assert.equal(fetches.filter((f) => f.id === other.id).length, MEMBER_PROBES_PER_DM); +}); + test('a rate limit error is not remembered as "not a member"', async (t) => { _resetTicketLookup(); t.mock.timers.enable({ apis: ['Date'], now: 4_000_000 }); @@ -159,11 +195,27 @@ test('a rate limit error is not remembered as "not a member"', async (t) => { }; const first = await lookupTicketGuilds(SENDER, clientOf([g])); assert.equal(first.incomplete, true); - t.mock.timers.tick(LOOKUP_REUSE_MS + 1); const second = await lookupTicketGuilds(SENDER, clientOf([g])); assert.equal(second.guilds.length, 1); }); +test('a server that refuses member lookups (403) is skipped for a while and does not keep the sender waiting', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 4_500_000 }); + const fetches = []; + const refusing = await ticketGuild(fetches, { refuse: true }); + const client = clientOf([refusing]); + const first = await lookupTicketGuilds(SENDER, client); + assert.deepEqual( + [first.guilds.length, first.incomplete], + [0, false], + 'a definite answer, not "send again"' + ); + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + await lookupTicketGuilds(SENDER, client); + assert.equal(fetches.length, 1, 'not asked again within the remember time'); +}); + test('canOpenTicketIn: cache, or the lookup that just ran, or one fetch', async (t) => { _resetTicketLookup(); t.mock.timers.enable({ apis: ['Date'], now: 5_000_000 }); @@ -183,44 +235,164 @@ test('canOpenTicketIn: cache, or the lookup that just ran, or one fetch', async assert.equal(await canOpenTicketIn(SENDER, cachedIn), false); }); -test('the DM handler asks the sender to try again when the lookup was cut short', async (t) => { +test('a platform that answers null for a non-member (and has no member cache) is handled', async () => { + _resetTicketLookup(); + const g = await ticketGuild([]); + g.members = { fetch: async (id) => (id === SENDER.id ? null : { id }) }; + const r = await lookupTicketGuilds(SENDER, clientOf([g])); + assert.deepEqual([r.guilds, r.incomplete], [[], false]); + const mine = await ticketGuild([]); + mine.members = { fetch: async (id) => ({ id }) }; + assert.equal((await lookupTicketGuilds({ id: '700000000000000077' }, clientOf([mine]))).guilds.length, 1); +}); + +// --- open tickets decide where a follow-up goes ------------------------------------------------- + +test('an open ticket is found from the database, beyond the per-DM cap and without the budgets', async (t) => { _resetTicketLookup(); t.mock.timers.enable({ apis: ['Date'], now: 6_000_000 }); + const fetches = []; const guilds = []; - for (let i = 0; i < 12; i++) guilds.push(await ticketGuild([])); - const replies = []; - const message = { + for (let i = 0; i < 40; i++) guilds.push(await ticketGuild(fetches)); + const x = await ticketGuild(fetches, { members: [SENDER.id] }); + guilds.push(x); + await createTicket(x.id, SENDER.id); + assert.deepEqual(await openTicketGuildKeys(SENDER.id), [x.id]); + + const r = await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.deepEqual(ids(r.openGuilds), [x.id]); + assert.equal(r.openComplete, true); + assert.equal(fetches[0].guild, x.id, 'the open ticket server is checked first'); + assert.equal(r.incomplete, true, 'the rest was cut short'); + + // Even with the user budget used up, the open ticket server is still found. + await lookupTicketGuilds(SENDER, clientOf(guilds)); + const again = await lookupTicketGuilds(SENDER, clientOf(guilds)); + assert.equal(again.incomplete, true); + assert.deepEqual(ids(again.openGuilds), [x.id]); +}); + +function dmFrom(user, guilds, replies) { + return { guild: null, partial: false, - content: 'I need help', - author: { id: '730000000000000001', bot: false }, + content: 'follow-up message', + author: { ...user, bot: false, tag: 'tester#0001', send: async () => {} }, attachments: new Map(), client: { ...clientOf(guilds), user: { id: '555555555555555555' } }, - reply: async (text) => replies.push(text), + reply: async (x) => replies.push(x), react: async () => {}, }; +} +const settle = async () => { + for (let i = 0; i < 30; i++) await new Promise((r) => setImmediate(r)); +}; + +test('a follow-up DM goes to the server with the open ticket, not to another server found first', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 7_000_000 }); + const user = { id: '740000000000000001' }; + const fetches = []; + const y = await ticketGuild(fetches, { cached: [user.id] }); // cached, first in cache order + const filler = []; + for (let i = 0; i < 14; i++) filler.push(await ticketGuild(fetches)); + const x = await ticketGuild(fetches, { members: [user.id] }); // not cached, beyond the cap + await createTicket(x.id, user.id); + const guilds = [y, ...filler, x]; + + const replies = []; const client = new EventEmitter(); register(client); - const settle = () => new Promise((r) => setImmediate(r)); - client.emit('messageCreate', message); - for (let i = 0; i < 20; i++) await settle(); + client.emit('messageCreate', dmFrom(user, guilds, replies)); + await settle(); + + const inX = await getOpenTicket(x.id, user.id); + assert.equal((await ticketMessages(inX.id)).length, 1, 'the follow-up is in the open ticket'); + assert.equal(await getOpenTicket(y.id, user.id), undefined, 'nothing was opened in the other server'); + assert.deepEqual(replies, []); +}); + +test('with no open ticket and a cut-short lookup, no ticket is opened: the member is asked to send again', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 8_000_000 }); + const user = { id: '740000000000000002' }; + const fetches = []; + const y = await ticketGuild(fetches, { cached: [user.id] }); + const filler = []; + for (let i = 0; i < 14; i++) filler.push(await ticketGuild(fetches)); + const guilds = [y, ...filler]; + + const replies = []; + const client = new EventEmitter(); + register(client); + client.emit('messageCreate', dmFrom(user, guilds, replies)); + await settle(); + assert.equal(await getOpenTicket(y.id, user.id), undefined, 'not opened in the only server found'); assert.equal(replies.length, 1); - assert.match(replies[0], /could not check all servers/); + assert.match(replies[0], /could not check all servers yet/); - t.mock.timers.tick(LOOKUP_REUSE_MS + 1); - client.emit('messageCreate', message); - for (let i = 0; i < 20; i++) await settle(); - assert.equal(replies.length, 2); - assert.match(replies[1], /not set up/, 'all servers checked by the second DM: a definite answer'); + // The retry continues with the servers not yet checked, and then goes through. + client.emit('messageCreate', dmFrom(user, guilds, replies)); + await settle(); + assert.ok(await getOpenTicket(y.id, user.id), 'all servers checked: now it opens in the one server'); }); -test('a platform that answers null for a non-member (and has no member cache) is handled', async () => { +test('a refused server pick keeps the stashed message and the menu', async (t) => { _resetTicketLookup(); - const g = await ticketGuild([]); - g.members = { fetch: async (id) => (id === SENDER.id ? null : { id }) }; - const r = await lookupTicketGuilds(SENDER, clientOf([g])); - assert.deepEqual(r, { guilds: [], incomplete: false }); - const mine = await ticketGuild([]); - mine.members = { fetch: async (id) => ({ id }) }; - assert.equal((await lookupTicketGuilds({ id: '700000000000000077' }, clientOf([mine]))).guilds.length, 1); + t.mock.timers.enable({ apis: ['Date'], now: 9_000_000 }); + const user = { id: '740000000000000003' }; + const fetches = []; + const a = await ticketGuild(fetches, { cached: [user.id] }); + const b = await ticketGuild(fetches, { cached: [user.id] }); + runtime.client = clientOf([a, b]); + t.after(() => { + runtime.client = null; + }); + const replies = []; + const client = new EventEmitter(); + register(client); + client.emit('messageCreate', dmFrom(user, [a, b], replies)); + await settle(); + assert.equal(replies.length, 1); + assert.ok(replies[0].components, 'the picker'); + + const sent = []; + const pick = (guild) => ({ + customId: 'ticket-guild', + values: [guild.id], + user: { ...user, tag: 'tester#0001', send: async () => {} }, + client: { guilds: { cache: new Map([[a.id, a]]) } }, + isRepliable: () => true, + update: async (x) => sent.push(['update', x]), + reply: async (x) => sent.push(['reply', x]), + }); + // A server the bot does not know (or cannot confirm): refused, nothing lost. + const ghost = { id: '999999999999999999' }; + await routeComponent(pick(ghost)); + assert.equal(sent.at(-1)[0], 'reply'); + assert.match(sent.at(-1)[1].content, /could not confirm/); + // Picking a real one afterwards still works: the message was kept. + await routeComponent(pick(a)); + assert.equal(sent.at(-1)[0], 'update'); + assert.match(sent.at(-1)[1].content, /Opened ticket/); +}); + +test('the DM handler asks the sender to try again when the lookup was cut short, then answers for real', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 10_000_000 }); + const user = { id: '740000000000000004' }; + const guilds = []; + for (let i = 0; i < 12; i++) guilds.push(await ticketGuild([])); + const replies = []; + const client = new EventEmitter(); + register(client); + client.emit('messageCreate', dmFrom(user, guilds, replies)); + await settle(); + assert.equal(replies.length, 1); + assert.match(replies[0], /could not check all servers/); + + client.emit('messageCreate', dmFrom(user, guilds, replies)); + await settle(); + assert.equal(replies.length, 2); + assert.match(replies[1], /not set up/, 'all servers checked by the second DM: a definite answer'); }); From 76c2a7aeaa637df9dddcafe4dae4f155eab4e6ae Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:54:25 +0100 Subject: [PATCH 08/21] fix(events): evict along the limit that was exceeded When the whole image cache was over its entry count, the holder with the most bytes gave up its entry, so tiny or failed entries of other servers could keep a real server from holding more than one image. The holder with the most entries now gives way when the entry limit is hit, and the one with the most bytes when the byte limit is hit. --- src/modules/lib/eventImage.js | 11 +++++++++-- test/eventImage.fairShare.test.js | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/modules/lib/eventImage.js b/src/modules/lib/eventImage.js index 5ac276d7..2b7946e8 100644 --- a/src/modules/lib/eventImage.js +++ b/src/modules/lib/eventImage.js @@ -145,8 +145,13 @@ function remember(url, entry) { if (victim === undefined) break; evict(victim); } - // Then the whole memory: the biggest holder gives up its oldest entry first. + // Then the whole memory: the holder with the most of whatever limit was hit gives up its oldest + // entry first. Too many entries: the one with the most entries (tiny images and remembered + // failures count there). Too many bytes: the one with the most bytes. Ranking by bytes when it + // is the entry limit that was hit would let tiny entries push out the one real image of another + // server. while (cache.size > MAX_ENTRIES || cachedBytes > MAX_TOTAL_BYTES) { + const byEntries = cache.size > MAX_ENTRIES; const held = new Map(); for (const e of cache.values()) { const h = held.get(e.owner) ?? { bytes: 0, entries: 0 }; @@ -154,7 +159,9 @@ function remember(url, entry) { h.entries += 1; held.set(e.owner, h); } - const order = [...held].sort((x, y) => y[1].bytes - x[1].bytes || y[1].entries - x[1].entries); + const key = byEntries ? 'entries' : 'bytes'; + const other = byEntries ? 'bytes' : 'entries'; + const order = [...held].sort((x, y) => y[1][key] - x[1][key] || y[1][other] - x[1][other]); const victim = order.map(([owner]) => oldestOf(owner, url)).find((k) => k !== undefined); if (victim === undefined) break; evict(victim); diff --git a/test/eventImage.fairShare.test.js b/test/eventImage.fairShare.test.js index f975ef93..b438af92 100644 --- a/test/eventImage.fairShare.test.js +++ b/test/eventImage.fairShare.test.js @@ -128,3 +128,21 @@ test('at most MAX_CONCURRENT_DOWNLOADS run at once across servers', async () => assert.equal(results.filter(Boolean).length, MAX_CONCURRENT_DOWNLOADS); assert.ok(await loadImage(url('after'), 'guild-x'), 'the slots are free again'); }); + +test('entry-count pressure takes from the server with the most entries, not the most bytes', async () => { + const HALF_MB = Buffer.concat([PNG_HEAD, Buffer.alloc(500 * 1024)]); + fetcher((link) => reply(link.includes('real') ? HALF_MB : SMALL)); + // Four servers fill their entry share with tiny images: the 64 entries of the whole cache. + for (const owner of ['a', 'b', 'c', 'd']) { + for (let n = 0; n < MAX_OWNER_ENTRIES; n += 1) await loadImage(url(`tiny-${owner}${n}`), owner); + } + assert.equal(_imageCacheSize().entries, MAX_ENTRIES); + // A real server then loads its icon and its event image. + await loadImage(url('real-icon'), 'guild-b'); + await loadImage(url('real-image'), 'guild-b'); + const before = calls.length; + assert.ok(await loadImage(url('real-icon'), 'guild-b')); + assert.ok(await loadImage(url('real-image'), 'guild-b')); + assert.equal(calls.length, before, 'both images of the real server stayed cached'); + assert.ok(_imageCacheSize().entries <= MAX_ENTRIES); +}); From d3f5f22ad91be39162fc892fe3474a69b453f54a Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:54:25 +0100 Subject: [PATCH 09/21] fix(backup): enforce the restore confirmation on the server, check superuser membership - POST /api/v2/health/backups/:name/restore: a confirm that is sent must be the snapshot name. An uploaded (sylo-imported-*) dump on Postgres needs { confirm: , acknowledgeSql: true }, else 400. Other snapshots need no body, so the Sylo app, which sends none, keeps working. The dashboard sends both after its warning. - The role check also reports membership of a superuser role (SET ROLE), and falls back to the shorter query only for an unknown role or function, never on other errors. - The docs link names the docs of the running version. --- docs/mobile-app.md | 2 +- src/db/backup.js | 29 +++++++++++++++ src/db/dbRole.js | 53 ++++++++++++++++++---------- src/web/routes/v2Api.js | 8 +++++ test/backup.test.js | 42 ++++++++++++++++++++++ test/dbRole.postgres.test.js | 45 +++++++++++++++++++++++ test/dbRole.test.js | 9 +++-- test/v2.guildAdmin.test.js | 14 ++++++++ web-v2/src/__tests__/Health.test.jsx | 20 ++++++++++- web-v2/src/api.js | 9 +++-- web-v2/src/pages/Health.jsx | 2 +- 11 files changed, 208 insertions(+), 25 deletions(-) diff --git a/docs/mobile-app.md b/docs/mobile-app.md index d03cd0c3..885c48fc 100644 --- a/docs/mobile-app.md +++ b/docs/mobile-app.md @@ -638,7 +638,7 @@ its account is an operator. | `POST /api/v2/health/dev-log-error-test` | Operators only | Logs a real error to test the pipeline; `{ ok }` | | `POST /api/v2/health/backups` | Operators only | Takes a backup; `{ name }` | | `GET /api/v2/health/backups/:name` | Operators only | The backup file, as a download | -| `POST /api/v2/health/backups/:name/delete`, `/restore` | Operators only | `{ ok }` (a restore then restarts the bot) | +| `POST /api/v2/health/backups/:name/delete`, `/restore` | Operators only | `{ ok }` (a restore then restarts the bot). `/restore` takes an optional `{ confirm: }`, which must match when sent. An uploaded (`sylo-imported-*`) dump on Postgres needs `{ confirm: , acknowledgeSql: true }` (its SQL runs with the rights of the database role), else 400 | | `POST /api/v2/health/backups/import` | Operators only | A raw body (`application/octet-stream`, up to 128 MB); `{ name }` | | `GET /api/v2/personalizer`, `/personalizer/presence` | Operators only | The bot; the presence and its choices | | `POST /api/v2/personalizer/identity`, `/presence`, `/announcement`, `/announcement/clear` | Operators only | Bot-wide settings | diff --git a/src/db/backup.js b/src/db/backup.js index 0a79f024..a9485d2c 100644 --- a/src/db/backup.js +++ b/src/db/backup.js @@ -236,6 +236,35 @@ export async function inspectDbFile(filePath) { return { ok: false, error: 'not a recognised Sylo backup file (SQLite snapshot or Postgres dump)' }; } +/** Was this snapshot uploaded (Health page import) rather than made by Sylo? */ +export const isImportedBackup = (name) => /^sylo-imported-/.test(String(name)); + +/** + * What a restore request must say before it is carried out, or null when it said enough. A + * `confirm` that is sent has to be the snapshot's name. An uploaded dump on Postgres is restored by + * running the SQL in it with the rights of the database role, so that one needs `confirm` and + * `acknowledgeSql: true`. Other snapshots need no body (the app sends none). + * @param {string} name + * @param {{ confirm?: unknown, acknowledgeSql?: unknown } | undefined} body + * @param {{ postgres: boolean }} o + * @returns {string | null} + */ +export function restoreConfirmationError(name, body, { postgres }) { + const { confirm, acknowledgeSql } = body ?? {}; + if (confirm !== undefined && confirm !== name) { + return 'confirm must be the name of the snapshot to restore'; + } + if (postgres && isImportedBackup(name)) { + if (confirm !== name) { + return 'Restoring an uploaded dump needs { "confirm": "", "acknowledgeSql": true }'; + } + if (acknowledgeSql !== true) { + return 'An uploaded Postgres dump runs its SQL with the rights of the database role: send "acknowledgeSql": true to restore it'; + } + } + return null; +} + /** * Validate an uploaded database and, if good, store it as a normal snapshot the * operator can then restore. Returns `{ ok, name?, error? }`. diff --git a/src/db/dbRole.js b/src/db/dbRole.js index 647d0db9..c794963d 100644 --- a/src/db/dbRole.js +++ b/src/db/dbRole.js @@ -9,9 +9,10 @@ // should have none of that: docs/postgres.md (Least-privilege database role). import { config } from '../config.js'; import { prepare } from './driver.js'; +import { BUILD, REPO_URL } from '../bot/lib/buildInfo.js'; -export const ROLE_DOCS_URL = - 'https://github.com/Ferdinand99/Sylo/blob/main/docs/postgres.md#least-privilege-database-role'; +// The docs of this version, so the link matches the behaviour the reader runs. +export const ROLE_DOCS_URL = `${REPO_URL}/blob/v${BUILD.version}/docs/postgres.md#least-privilege-database-role`; /** * @typedef {object} RolePrivileges @@ -22,6 +23,7 @@ export const ROLE_DOCS_URL = * @property {boolean} execProgram member of pg_execute_server_program * @property {boolean} readFiles member of pg_read_server_files * @property {boolean} writeFiles member of pg_write_server_files + * @property {boolean} superMember member of a superuser role (it can SET ROLE to it) */ const yes = (v) => v === true || v === 1 || v === 't'; @@ -36,6 +38,7 @@ export function roleRisks(p) { if (p.execProgram) risks.push('pg_execute_server_program'); if (p.readFiles) risks.push('pg_read_server_files'); if (p.writeFiles) risks.push('pg_write_server_files'); + if (p.superMember) risks.push('membership of a superuser role'); return risks; } @@ -60,12 +63,35 @@ const FULL = ` SELECT current_user AS name, r.rolsuper, r.rolcreaterole, r.rolcreatedb, pg_has_role(current_user, 'pg_execute_server_program', 'MEMBER') AS exec_program, pg_has_role(current_user, 'pg_read_server_files', 'MEMBER') AS read_files, - pg_has_role(current_user, 'pg_write_server_files', 'MEMBER') AS write_files + pg_has_role(current_user, 'pg_write_server_files', 'MEMBER') AS write_files, + EXISTS (SELECT 1 FROM pg_roles s WHERE s.rolsuper AND pg_has_role(current_user, s.oid, 'MEMBER')) AS super_member FROM pg_roles r WHERE r.rolname = current_user`; -// Servers before Postgres 11 have no predefined roles to ask about. +// Servers before Postgres 11 have no predefined roles to ask about. Only that case (an unknown +// role or function) uses this shorter question; any other error means "could not tell", never +// "no extra rights". const BASIC = ` - SELECT current_user AS name, r.rolsuper, r.rolcreaterole, r.rolcreatedb + SELECT current_user AS name, r.rolsuper, r.rolcreaterole, r.rolcreatedb, + EXISTS (SELECT 1 FROM pg_roles s WHERE s.rolsuper AND pg_has_role(current_user, s.oid, 'MEMBER')) AS super_member FROM pg_roles r WHERE r.rolname = current_user`; +const UNDEFINED_OBJECT_OR_FUNCTION = new Set(['42704', '42883']); + +/** The answer of FULL or BASIC as RolePrivileges (null for no row). */ +export function privilegesFromRow(row) { + if (!row) return null; + return { + name: String(row.name), + superuser: yes(row.rolsuper), + createRole: yes(row.rolcreaterole), + createDb: yes(row.rolcreatedb), + execProgram: yes(row.exec_program), + readFiles: yes(row.read_files), + writeFiles: yes(row.write_files), + superMember: yes(row.super_member), + }; +} + +/** The two questions, for tests that ask them over a connection of their own. */ +export const ROLE_QUERIES = { full: FULL, basic: BASIC }; /** * Read the privileges of the connected role. Postgres only; resolves to null on SQLite or when the @@ -76,19 +102,10 @@ export async function readRolePrivileges() { if (!config.databaseUrl) return null; for (const sql of [FULL, BASIC]) { try { - const row = await prepare(sql).get(); - if (!row) return null; - return { - name: String(row.name), - superuser: yes(row.rolsuper), - createRole: yes(row.rolcreaterole), - createDb: yes(row.rolcreatedb), - execProgram: yes(row.exec_program), - readFiles: yes(row.read_files), - writeFiles: yes(row.write_files), - }; - } catch { - // try the simpler question + return privilegesFromRow(await prepare(sql).get()); + } catch (err) { + if (!UNDEFINED_OBJECT_OR_FUNCTION.has(err?.code)) return null; // could not tell + // an older server: try the simpler question } } return null; diff --git a/src/web/routes/v2Api.js b/src/web/routes/v2Api.js index a527004b..5b448d88 100644 --- a/src/web/routes/v2Api.js +++ b/src/web/routes/v2Api.js @@ -330,6 +330,7 @@ import { openBackup, isBackupName, inspectDbFile, + restoreConfirmationError, } from '../../db/backup.js'; import { roleRisks, lastRolePrivileges, ROLE_DOCS_URL } from '../../db/dbRole.js'; import { offsiteBackupStatus } from '../../db/offsiteBackup.js'; @@ -2908,6 +2909,13 @@ router.post( const name = req.params.name; const full = resolveBackup(name); if (!full) return res.status(404).json({ error: 'no such backup' }); + // A confirmation that is sent has to be the right one. An uploaded Postgres dump is restored by + // running the SQL in it with the rights of the database role, so for that one the request must + // name the snapshot and say that it knows (the dashboard sends both after its own warning). + // Other snapshots stay restorable without a body, as the app sends it. + const problem = restoreConfirmationError(name, req.body, { postgres: Boolean(config.databaseUrl) }); + if (problem) return res.status(400).json({ error: problem }); + const check = await inspectDbFile(full); if (!check.ok) return res.status(400).json({ error: check.error }); diff --git a/test/backup.test.js b/test/backup.test.js index 6380316c..6d8c0f18 100644 --- a/test/backup.test.js +++ b/test/backup.test.js @@ -112,3 +112,45 @@ test('importBuffer stores a valid upload as a snapshot and rejects bad input', a assert.equal((await importBuffer(Buffer.alloc(2000))).ok, false); // right size, wrong magic assert.equal((await importBuffer('not a buffer')).ok, false); }); + +// --- what a restore request must confirm ----------------------------------------------------- + +test('restoreConfirmationError: an uploaded Postgres dump needs the name and the acknowledgement', async () => { + const { restoreConfirmationError, isImportedBackup } = await import('../src/db/backup.js'); + const imported = 'sylo-imported-2026-11-16-10-00-00-000.dump'; + const own = 'sylo-manual-2026-11-16-10-00-00-000.dump'; + assert.equal(isImportedBackup(imported), true); + assert.equal(isImportedBackup(own), false); + + // Postgres, uploaded: both fields, each checked. + assert.match(restoreConfirmationError(imported, undefined, { postgres: true }), /needs/); + assert.match(restoreConfirmationError(imported, {}, { postgres: true }), /needs/); + assert.match( + restoreConfirmationError(imported, { confirm: imported }, { postgres: true }), + /acknowledgeSql/ + ); + assert.match( + restoreConfirmationError(imported, { confirm: imported, acknowledgeSql: 'true' }, { postgres: true }), + /acknowledgeSql/, + 'only a real true' + ); + assert.match( + restoreConfirmationError(imported, { confirm: 'other', acknowledgeSql: true }, { postgres: true }), + /name of the snapshot/ + ); + assert.equal( + restoreConfirmationError(imported, { confirm: imported, acknowledgeSql: true }, { postgres: true }), + null + ); + + // Not uploaded, or SQLite: no body needed (the app sends none), but a wrong confirm is refused. + for (const postgres of [true, false]) { + assert.equal(restoreConfirmationError(own, undefined, { postgres }), null); + assert.equal(restoreConfirmationError(own, { confirm: own }, { postgres }), null); + assert.match(restoreConfirmationError(own, { confirm: imported }, { postgres }), /name of the snapshot/); + } + assert.equal( + restoreConfirmationError(imported.replace('.dump', '.db'), undefined, { postgres: false }), + null + ); +}); diff --git a/test/dbRole.postgres.test.js b/test/dbRole.postgres.test.js index 448a2b96..6c28f93d 100644 --- a/test/dbRole.postgres.test.js +++ b/test/dbRole.postgres.test.js @@ -37,3 +37,48 @@ test( if (warned.length) assert.match(warned[0][1], /docs\/postgres\.md#least-privilege-database-role/); } ); + +test( + 'dbRole queries tell a role without extra rights from one that has them', + { skip: !url && 'DATABASE_URL not set (sqlite-only run)' }, + async (t) => { + const { ROLE_QUERIES, privilegesFromRow, roleRisks } = await import('../src/db/dbRole.js'); + const postgres = (await import('postgres')).default; + const admin = postgres(url); + t.after(() => admin.end({ timeout: 1 })); + const [me] = + await admin`SELECT current_user AS name, rolsuper, rolcreaterole FROM pg_roles WHERE rolname = current_user`; + if (!me.rolsuper && !me.rolcreaterole) return t.skip('the test role cannot create roles'); + + const role = 'sylo_test_lowpriv'; + await admin.unsafe(`DROP ROLE IF EXISTS ${role}`); + await admin.unsafe(`CREATE ROLE ${role} LOGIN PASSWORD 'lowpriv' NOSUPERUSER NOCREATEDB NOCREATEROLE`); + const asRole = new URL(url); + asRole.username = role; + asRole.password = 'lowpriv'; + const ask = async () => { + const low = postgres(asRole.toString()); + try { + return privilegesFromRow((await low.unsafe(ROLE_QUERIES.full))[0]); + } finally { + await low.end({ timeout: 1 }); + } + }; + try { + assert.deepEqual(roleRisks(await ask()), [], 'a plain login role has no extra rights'); + + await admin.unsafe(`GRANT pg_execute_server_program TO ${role}`); + assert.deepEqual(roleRisks(await ask()), ['pg_execute_server_program']); + + if (me.rolsuper) { + // It is not a superuser itself, but may become one. + await admin.unsafe(`GRANT ${me.name} TO ${role}`); + assert.ok(roleRisks(await ask()).includes('membership of a superuser role')); + } + await admin.unsafe(`ALTER ROLE ${role} CREATEDB`); + assert.ok(roleRisks(await ask()).includes('CREATEDB')); + } finally { + await admin.unsafe(`DROP ROLE IF EXISTS ${role}`); + } + } +); diff --git a/test/dbRole.test.js b/test/dbRole.test.js index 6bfc2512..17232339 100644 --- a/test/dbRole.test.js +++ b/test/dbRole.test.js @@ -18,6 +18,7 @@ const plain = { execProgram: false, readFiles: false, writeFiles: false, + superMember: false, }; test('a role with no extra rights gets no warning', () => { @@ -34,6 +35,7 @@ test('superuser, CREATEROLE and CREATEDB each warn, naming the right and linking ['execProgram', 'pg_execute_server_program'], ['readFiles', 'pg_read_server_files'], ['writeFiles', 'pg_write_server_files'], + ['superMember', 'membership of a superuser role'], ]) { const warning = rolePrivilegeWarning({ ...plain, [key]: true }); assert.match(warning, new RegExp(label), key); @@ -44,8 +46,11 @@ test('superuser, CREATEROLE and CREATEDB each warn, naming the right and linking assert.deepEqual(roleRisks({ ...plain, superuser: true, createDb: true }), ['SUPERUSER', 'CREATEDB']); }); -test('the docs link points at the least-privilege section of docs/postgres.md', () => { - assert.match(ROLE_DOCS_URL, /docs\/postgres\.md#least-privilege-database-role$/); +test('the docs link points at the least-privilege section of the docs of this version', () => { + assert.match( + ROLE_DOCS_URL, + /\/blob\/v\d+\.\d+\.\d+[^/]*\/docs\/postgres\.md#least-privilege-database-role$/ + ); }); test('on SQLite there is nothing to read and nothing to warn about', async () => { diff --git a/test/v2.guildAdmin.test.js b/test/v2.guildAdmin.test.js index ace90eb5..b6900f22 100644 --- a/test/v2.guildAdmin.test.js +++ b/test/v2.guildAdmin.test.js @@ -253,6 +253,20 @@ test('snapshot names that are not snapshot file names are refused, and nothing o } }); +test('a restore that sends a confirmation must name the snapshot: a wrong one is refused and nothing is restored', async () => { + const name = 'sylo-manual-2026-01-01-00-00-00-000.db'; + const file = join(backupDir(), name); + writeFileSync(file, 'not a database'); + try { + const wrong = await api.postRoot(`/health/backups/${name}/restore`, { confirm: 'sylo-manual-other.db' }); + assert.equal(wrong.status, 400); + assert.match((await wrong.json()).error, /name of the snapshot/); + assert.ok(existsSync(file), 'still there, and this process is still running'); + } finally { + rmSync(file, { force: true }); + } +}); + // --- composed messages and tickets ------------------------------------------------------------ // replaces routes.misc.test.js: "POST /guilds/:id/messages/new saves a draft" diff --git a/web-v2/src/__tests__/Health.test.jsx b/web-v2/src/__tests__/Health.test.jsx index 8d60bf61..e5d59c1d 100644 --- a/web-v2/src/__tests__/Health.test.jsx +++ b/web-v2/src/__tests__/Health.test.jsx @@ -15,7 +15,7 @@ vi.mock('../api.js', () => ({ backupDownloadUrl: vi.fn(), ApiError: class ApiError extends Error {}, })); -const { getHealth } = await import('../api.js'); +const { getHealth, restoreBackup } = await import('../api.js'); const { confirmDialog } = await import('../components/ConfirmDialog.jsx'); function health(sessionSecret) { @@ -121,6 +121,24 @@ describe('Health: restoring an uploaded Postgres dump', () => { expect(container.querySelector('a')?.getAttribute('href')).toMatch(/least-privilege-database-role/); }); + it.each([ + [0, 'sylo-imported-20261116.dump', true], + [1, 'sylo-20261116.dump', false], + ])( + 'restore button %i sends the name, with acknowledgeSql %s only for an uploaded dump', + async (index, name, ack) => { + getHealth.mockResolvedValue(pgHealth()); + restoreBackup.mockResolvedValue({ ok: true }); + confirmDialog.mockResolvedValue(true); + render(); + await screen.findByText(name); + fireEvent.click(restoreButtons()[index]); + await waitFor(() => expect(restoreBackup).toHaveBeenCalledTimes(1)); + expect(restoreBackup).toHaveBeenLastCalledWith(name, { acknowledgeSql: ack }); + confirmDialog.mockResolvedValue(false); + } + ); + it('does not add that warning for a backup Sylo made itself', async () => { getHealth.mockResolvedValue(pgHealth()); render(); diff --git a/web-v2/src/api.js b/web-v2/src/api.js index e93b45e1..f82918a2 100644 --- a/web-v2/src/api.js +++ b/web-v2/src/api.js @@ -302,8 +302,13 @@ export const sendDevLogErrorTest = () => postJson('/api/v2/health/dev-log-error- export const createBackup = () => postJson('/api/v2/health/backups', {}); export const deleteBackup = (name) => postJson(`/api/v2/health/backups/${encodeURIComponent(name)}/delete`, {}); -export const restoreBackup = (name) => - postJson(`/api/v2/health/backups/${encodeURIComponent(name)}/restore`, {}); +// `acknowledgeSql`: the person was told that an uploaded Postgres dump runs its SQL with the rights +// of the database role (the server requires it for those). `confirm` names the snapshot. +export const restoreBackup = (name, { acknowledgeSql = false } = {}) => + postJson(`/api/v2/health/backups/${encodeURIComponent(name)}/restore`, { + confirm: name, + ...(acknowledgeSql ? { acknowledgeSql: true } : {}), + }); export const importBackup = (file) => apiFetch('/api/v2/health/backups/import', { method: 'POST', diff --git a/web-v2/src/pages/Health.jsx b/web-v2/src/pages/Health.jsx index f74c2b71..608f3fcf 100644 --- a/web-v2/src/pages/Health.jsx +++ b/web-v2/src/pages/Health.jsx @@ -211,7 +211,7 @@ export default function Health() { }); if (!ok) return; const result = await run( - () => restoreBackup(name), + () => restoreBackup(name, { acknowledgeSql: Boolean(d.db.postgres && isImportedBackup(name)) }), () => 'Restoring…' ); if (result?.ok) setRestoring(name); From a7d037d72867dfe163e21d95aba6ad81c7ebeece Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:54:25 +0100 Subject: [PATCH 10/21] test(tickets): make the ported access guard tests sharper The central error handler is exported from server.js instead of being found as the first four-argument layer, and the lookup failure is tested with a real server instead of a missing params object. --- src/web/server.js | 60 ++++++++++++++++++++------------------ test/classicBanner.test.js | 48 +++++++++++++++++++----------- 2 files changed, 62 insertions(+), 46 deletions(-) diff --git a/src/web/server.js b/src/web/server.js index ed602ba0..05d3ec45 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -135,6 +135,36 @@ export function routeLabel(req) { return `${mount}${req.route?.path ?? '(unmatched)'}`; } +// Central error handler — keep the server up, record the error for the dashboard. +// Express recognises an error handler by its 4-arg signature, so `_next` must +// be present even though it is unused. +export function centralErrorHandler(err, req, res, _next) { + // The person's own doing, not a server error: a per-server ceiling (src/lib/limits.js) or a + // role they may not hand out (src/web/middleware/roleGrants.js). The reason reaches them as + // JSON or an error page. + if (err instanceof LimitError || err instanceof RoleGrantError) { + const status = err.status; + const heading = err instanceof LimitError ? 'Limit reached' : 'Role not allowed'; + if (req.originalUrl.startsWith('/api/') || !req.accepts('html')) { + return res.status(status).json({ error: err.message }); + } + return res.status(status).render('error', { title: heading, heading, message: err.message }); + } + // A request the client got wrong, not a server fault: a body over the cap (413) or one that + // does not parse (400), from the body parsers. Anyone can send these, so they must not reach + // log.error (dev-log channel, operators' phones, /health): answer and move on. + const status = err?.status ?? err?.statusCode; + if (err?.expose === true && Number.isInteger(status) && status >= 400 && status < 500) { + log.debug('web', `${req.method} ${routeLabel(req)} ${status} ${err.type ?? ''}`.trim()); + if (res.headersSent) return res.end(); + return res.status(status).json({ error: status === 413 ? 'Payload too large' : 'Bad request' }); + } + // The route pattern, never the raw path: the path is the caller's to choose (a unique message + // per request) and in a webhook URL it holds a secret token. + log.error('web', `${req.method} ${routeLabel(req)}`, err); + res.status(500).json({ error: 'Internal server error' }); +} + /** Build (but do not start) the Express app. Exposed for testing. */ export function createApp() { const app = express(); @@ -286,35 +316,7 @@ export function createApp() { }); }); - // Central error handler — keep the server up, record the error for the dashboard. - // Express recognises an error handler by its 4-arg signature, so `_next` must - // be present even though it is unused. - app.use((err, req, res, _next) => { - // The person's own doing, not a server error: a per-server ceiling (src/lib/limits.js) or a - // role they may not hand out (src/web/middleware/roleGrants.js). The reason reaches them as - // JSON or an error page. - if (err instanceof LimitError || err instanceof RoleGrantError) { - const status = err.status; - const heading = err instanceof LimitError ? 'Limit reached' : 'Role not allowed'; - if (req.originalUrl.startsWith('/api/') || !req.accepts('html')) { - return res.status(status).json({ error: err.message }); - } - return res.status(status).render('error', { title: heading, heading, message: err.message }); - } - // A request the client got wrong, not a server fault: a body over the cap (413) or one that - // does not parse (400), from the body parsers. Anyone can send these, so they must not reach - // log.error (dev-log channel, operators' phones, /health): answer and move on. - const status = err?.status ?? err?.statusCode; - if (err?.expose === true && Number.isInteger(status) && status >= 400 && status < 500) { - log.debug('web', `${req.method} ${routeLabel(req)} ${status} ${err.type ?? ''}`.trim()); - if (res.headersSent) return res.end(); - return res.status(status).json({ error: status === 413 ? 'Payload too large' : 'Bad request' }); - } - // The route pattern, never the raw path: the path is the caller's to choose (a unique message - // per request) and in a webhook URL it holds a secret token. - log.error('web', `${req.method} ${routeLabel(req)}`, err); - res.status(500).json({ error: 'Internal server error' }); - }); + app.use(centralErrorHandler); return app; } diff --git a/test/classicBanner.test.js b/test/classicBanner.test.js index dd383035..0b84efc3 100644 --- a/test/classicBanner.test.js +++ b/test/classicBanner.test.js @@ -11,7 +11,7 @@ import express from 'express'; await startOAuthApp(); const { pkceChallenge } = await import('../src/db/apiTokens.js'); -const { createApp } = await import('../src/web/server.js'); +const { createApp, centralErrorHandler } = await import('../src/web/server.js'); const { forbidOwner } = await import('../src/web/middleware/auth.js'); const { ticketAccessLevel } = await import('../src/web/middleware/ticketAccess.js'); const { ticketsRouter } = await import('../src/web/routes/v2Staff.js'); @@ -136,8 +136,7 @@ test('the error page from the operator guard (auth.js) is plain', async () => { }); test('the error page from the server error handler (server.js) is plain', async () => { - const handler = app.router.stack.map((l) => l.handle).find((h) => h.length === 4); - assert.ok(handler, 'the central error handler'); + const handler = centralErrorHandler; for (const err of [ new LimitError('You reached the limit.'), new RoleGrantError('That role is above yours.'), @@ -164,8 +163,7 @@ test('the error page renders with and without a title, heading and message', asy // The ticket access guard (ticketAccess.js) is the first thing the tickets router of the v2 API runs // (v2Staff.js). This mounts that router behind the server's own central error handler, with a // signed-in account chosen per request, to test what the guard answers. -async function withTicketsRouter(run) { - const central = app.router.stack.map((l) => l.handle).find((h) => h.length === 4); +async function withTicketsRouter(run, { seen = [] } = {}) { const mini = express(); mini.use((req, _res, next) => { req.account = @@ -180,7 +178,10 @@ async function withTicketsRouter(run) { next(); }); mini.use('/guilds/:guildId/tickets', ticketsRouter); - mini.use(central); + mini.use((err, req, res, next) => { + seen.push(err); + centralErrorHandler(err, req, res, next); + }); const server = await new Promise((resolve) => { const srv = mini.listen(0, () => resolve(srv)); }); @@ -208,17 +209,30 @@ test('the ticket access guard refuses a person without access with a plain answe test('a failed ticket access lookup goes to the error handler instead of hanging the request', async (t) => { const logged = t.mock.method(log, 'error', () => {}); - await ticketAccessLevel({ account: { id: 'acc-1', identities: [] } }).then( - () => assert.fail('a lookup without a server must not turn into a level'), - (err) => assert.ok(err instanceof Error, 'the lookup rejects, it is not swallowed into a level') + const failing = { + id: 'acc-1', + get identities() { + throw new Error('lookup failed: secret detail'); + }, + }; + await assert.rejects( + ticketAccessLevel({ account: failing, params: { guildId: D_GUILD }, method: 'GET' }), + /lookup failed/, + 'the lookup rejects: a failure is not swallowed into a level' ); - await withTicketsRouter(async (url) => { - const res = await fetch(url, { - headers: { accept: 'text/html', 'x-lookup': 'fails' }, - signal: AbortSignal.timeout(5000), - }); - assert.equal(res.status, 500, 'answered, not hung'); - assert.deepEqual(await res.json(), { error: 'Internal server error' }, 'no detail of the failure'); - }); + const seen = []; + await withTicketsRouter( + async (url) => { + const res = await fetch(url, { + headers: { accept: 'text/html', 'x-lookup': 'fails' }, + signal: AbortSignal.timeout(5000), + }); + assert.equal(res.status, 500, 'answered, not hung'); + assert.deepEqual(await res.json(), { error: 'Internal server error' }, 'no detail of the failure'); + }, + { seen } + ); + assert.equal(seen.length, 1, 'the error reached the central error handler'); + assert.match(seen[0].message, /lookup failed/); assert.equal(logged.mock.callCount(), 1, 'recorded once, with the route pattern'); }); From ad4059e00f0878af320aaea622f7835f03171d96 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 01:54:32 +0100 Subject: [PATCH 11/21] docs(postgres): the server asks for the acknowledgement of an uploaded dump too --- docs/postgres.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/postgres.md b/docs/postgres.md index 4e650535..651e99b4 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -162,9 +162,10 @@ SELECT pg_has_role(current_user, 'pg_execute_server_program', 'MEMBER') AS exec_ pg_has_role(current_user, 'pg_write_server_files', 'MEMBER') AS write_files; ``` -Every value should be `f`. Restores are never blocked because of the role; the Health page asks -you to confirm before it restores an uploaded dump and says that its SQL runs with the role's -rights. Even with a least-privilege role, a restored dump can still do anything that role may do +Every value should be `f`. Restores are never blocked because of the role. Before it restores an +uploaded dump the Health page says that its SQL runs with the role's rights, and the server does +not restore one unless the request names the snapshot (`confirm`) and sets `acknowledgeSql: true` +(the dashboard sends both after its warning; other clients of the API must do the same). Even with a least-privilege role, a restored dump can still do anything that role may do inside the Sylo database (create functions and triggers, change or delete any row), so restore only files you made or trust. From 6b649e89f678b37a64912a614d8d02c0fe528772 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 02:20:06 +0100 Subject: [PATCH 12/21] fix(live-alerts): poll every batch first, search only the failed ones, check with a canary - Phase one sends one request per batch, so a clean batch is never left out because poisoned values filled an earlier one, and each clean batch is evidence that the request is fine. - Phase two halves only the failed batches, in turn, within its own budget. Running out of budget leaves the unreached items for the next poll and never fails the poll. The start of the list rotates, so the same servers are not always last. - Before values are quarantined, one request with a value accepted in an earlier poll tells a refused value from a refusing platform (outage, flapping): if it is refused too, nothing is quarantined, and the poll fails only when nothing succeeded. With nothing known to be good yet and nothing succeeding, nothing is quarantined and the poll says so. --- src/modules/kickAlerts.js | 14 ++- src/modules/lib/batchIsolation.js | 154 ++++++++++++++++++++--------- src/modules/twitchAlerts.js | 12 ++- test/liveAlertsIsolation.test.js | 159 +++++++++++++++++++++++++++++- 4 files changed, 283 insertions(+), 56 deletions(-) diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index f127ae83..ee109955 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -111,15 +111,15 @@ export const rejectedSlugs = createQuarantine(); /** * Live channels for a set of slugs, keyed by lowercased slug. Only currently-live * channels are included. A slug that Kick rejects is isolated (see fetchIsolated) and left out. - * `report({ rejected, unresolved })` is called once with the slugs rejected now and every slug + * `report({ rejected, unresolved, inconclusive })` is called once with the slugs rejected now and every slug * this poll knows nothing about. */ export async function fetchLiveChannels(slugs, { report } = {}) { const out = new Map(); - const { results, rejected, unresolved } = await fetchIsolated(slugs, 50, getChannels, { + const { results, rejected, unresolved, inconclusive } = await fetchIsolated(slugs, 50, getChannels, { quarantine: rejectedSlugs, }); - report?.({ rejected, unresolved }); + report?.({ rejected, unresolved, inconclusive }); for (const ch of results) { if (ch?.stream?.is_live && ch.slug) out.set(String(ch.slug).toLowerCase(), ch); } @@ -195,8 +195,14 @@ export async function tick() { let unresolved = new Set(); try { live = await fetchLiveChannels(slugs, { - report: ({ rejected, unresolved: left }) => { + report: ({ rejected, unresolved: left, inconclusive }) => { unresolved = left; + if (inconclusive) { + log.warn( + 'kick-alerts', + 'Every request of this poll was refused and no earlier accepted value is known to check against, so nothing was quarantined.' + ); + } const warning = rejectedWarning('Kick slugs', rejected, (slug) => [ ...new Set(jobs.filter((j) => j.alert.slug === slug).map((j) => j.guildId)), ]); diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js index dd14c67a..bb059f34 100644 --- a/src/modules/lib/batchIsolation.js +++ b/src/modules/lib/batchIsolation.js @@ -1,23 +1,27 @@ -// Isolate one rejected item from a shared, batched platform request. +// Isolate rejected items from a shared, batched platform request. // // The Twitch and Kick alert modules poll every server's streamers together. If the platform -// answers a batch with a 4xx because of one stored login or slug, that single value must not stop -// the poll for every other server. fetchIsolated() splits such a batch in half until the rejected -// item stands alone, and keeps the results of the rest. After the poll the rejected items are -// quarantined with a growing backoff, so the next polls leave them out. +// answers a batch with a 4xx because of stored logins or slugs, those values must not stop the +// poll for every other server. fetchIsolated() works in two phases: +// 1. one request per batch. A batch that answers is done, and counts as evidence that the request +// itself is fine. This always fits the budget, so a clean batch is never left out because +// another one needed a long search. +// 2. only the batches that failed are halved until the rejected items stand alone. This has its +// own budget (EXTRA_REQUESTS), spread over the failing batches in turn. When it runs out, the +// items of the halves not reached are left out of this poll (never failed): the next poll, +// with the found values already quarantined and the batches cut differently (the start +// rotates), reaches them. A poll never fails because the budget ran out. +// The values found are quarantined with a growing backoff after the poll. // -// Item fault or request fault? The poll's own evidence decides. If at least one request of the -// poll succeeded, an item that is rejected on its own is at fault, however many there are. If no -// request of the poll succeeded, the request is at fault (bad credentials, a blocked address, an -// outage): nothing is quarantined and the poll fails like any other failed poll. Only the -// statuses that can mean "this value" count (400, 404, 422); 5xx, network errors, 401, 429 and the -// other 4xx fail the poll. -// -// The cost is capped: a poll makes at most `batches + EXTRA_REQUESTS` requests. When that runs -// out, the batches not yet resolved are left out of this poll (not failed); each poll quarantines -// the values it found, so the next poll resolves more. +// Item fault or request fault? Only the statuses that can mean "this value" count (400, 404, 422); +// 5xx, network errors, 401, 429 and the other 4xx fail the poll. Before values are quarantined, one +// canary request is sent with a value that an earlier poll saw accepted. If the canary fails the +// same way, the platform is refusing everything (bad credentials, a blocked address, an outage, +// flapping): nothing is quarantined, and when no request of the poll succeeded the poll fails like +// any other failed poll. If no value is known to be good yet and nothing succeeded, the poll cannot +// tell: nothing is quarantined and it reports `inconclusive`. -/** Requests per poll beyond one per batch, for finding rejected items. */ +/** Requests per poll, beyond one per batch, for finding the rejected items. */ export const EXTRA_REQUESTS = 120; /** Statuses that can mean "this value was refused", as against a problem with the request. */ @@ -38,6 +42,8 @@ export function createQuarantine({ } = {}) { /** @type {Map} */ const entries = new Map(); + let good = null; + let turn = 0; return { /** True while the item is waiting out its backoff. */ has(key) { @@ -66,53 +72,61 @@ export function createQuarantine({ get size() { return entries.size; }, + /** A value that the platform accepted in an earlier poll (the canary), or null. */ + get lastGood() { + return good; + }, + setGood(item) { + good = item; + }, + /** Where the next poll starts in its list of `length` items: a different place each time. */ + nextStart(length, step) { + const start = length > 0 ? turn % length : 0; + turn += step; + return start; + }, }; } /** - * Run `fetchBatch` over `items` in chunks of `size`, bisecting any chunk the platform rejects. + * Run `fetchBatch` over `items` in chunks of `size`, halving the chunks the platform rejects. * * @template T * @param {string[]} items * @param {number} size * @param {(batch: string[]) => Promise} fetchBatch throws an error with `.status` on a non-OK answer - * @param {{ quarantine: ReturnType, maxRequests?: number }} o + * @param {{ quarantine: ReturnType, extraRequests?: number }} o * @returns {Promise<{ * results: T[], * rejected: Array<{ item: string, err: Error, waitMs: number, strikes: number }>, * unresolved: Set, + * inconclusive: boolean, * }>} `unresolved`: the items this poll knows nothing about (quarantined before, rejected now, or - * not reached when the request budget ran out); their stored state must be left alone. + * not reached when the budget ran out); their stored state must be left alone. */ -export async function fetchIsolated(items, size, fetchBatch, { quarantine, maxRequests }) { +export async function fetchIsolated(items, size, fetchBatch, { quarantine, extraRequests = EXTRA_REQUESTS }) { const results = []; const unresolved = new Set(); const found = []; // rejected on their own: { item, err } - let requests = 0; + const canary = quarantine.lastGood; let succeeded = 0; - const budget = maxRequests ?? Math.ceil(items.length / size) + EXTRA_REQUESTS; + let lastOk = null; - const run = async (batch) => { - if (requests >= budget) { - for (const item of batch) unresolved.add(item); - return; - } - requests += 1; + /** @returns {Promise} true when the platform answered; the error for a refused value */ + const ask = async (batch, keepResults = true) => { try { - results.push(...(await fetchBatch(batch))); + const got = await fetchBatch(batch); + if (keepResults) results.push(...got); } catch (err) { if (!isRejectedStatus(err?.status)) throw err; - if (batch.length === 1) { - found.push({ item: batch[0], err }); - return; - } - const mid = Math.ceil(batch.length / 2); - await run(batch.slice(0, mid)); - await run(batch.slice(mid)); - return; + return err; } - succeeded += 1; - for (const item of batch) quarantine.clear(item); + if (keepResults) { + succeeded += 1; + lastOk = batch[0]; + for (const item of batch) quarantine.clear(item); + } + return true; }; const live = items.filter((item) => { @@ -120,17 +134,63 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, maxRe unresolved.add(item); return false; }); - for (let i = 0; i < live.length; i += size) await run(live.slice(i, i + size)); + const start = quarantine.nextStart(live.length, size); + const order = [...live.slice(start), ...live.slice(0, start)]; - // Nothing succeeded: the request is at fault, not the items. Quarantine nothing. - if (found.length > 0 && succeeded === 0) throw found[0].err; + // Phase 1: one request per batch. + const queue = []; + for (let i = 0; i < order.length; i += size) { + const batch = order.slice(i, i + size); + const answer = await ask(batch); + if (answer === true) continue; + if (batch.length === 1) found.push({ item: batch[0], err: answer }); + else queue.push(batch); + } - const rejected = found.map(({ item, err }) => { - const waitMs = quarantine.add(item); + // Phase 2: halve the failed batches, in turn, until the budget is used up. + let spare = extraRequests; + while (queue.length > 0) { + const batch = queue.shift(); + const mid = Math.ceil(batch.length / 2); + for (const half of [batch.slice(0, mid), batch.slice(mid)]) { + if (spare <= 0) { + for (const item of half) unresolved.add(item); + continue; + } + spare -= 1; + const answer = await ask(half); + if (answer === true) continue; + if (half.length === 1) found.push({ item: half[0], err: answer }); + else queue.push(half); + } + } + + let inconclusive = false; + let quarantineFound = found.length > 0; + if (quarantineFound) { + if (canary && !found.some((f) => f.item === canary)) { + // Is the platform refusing everything? One request with a value that was fine before. + const answer = await ask([canary], false); + if (answer !== true) { + quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time + if (succeeded === 0) throw found[0].err; + quarantineFound = false; // flapping: some requests answer, some are refused + } + } else if (succeeded === 0) { + inconclusive = true; // nothing answered and nothing to compare with + quarantineFound = false; + } + } + if (lastOk !== null) quarantine.setGood(lastOk); + + const rejected = []; + for (const { item, err } of found) { unresolved.add(item); - return { item, err, waitMs, strikes: quarantine.strikes(item) }; - }); - return { results, rejected, unresolved }; + if (!quarantineFound) continue; + const waitMs = quarantine.add(item); + rejected.push({ item, err, waitMs, strikes: quarantine.strikes(item) }); + } + return { results, rejected, unresolved, inconclusive }; } /** diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 6fffeca6..04a5aa4d 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -110,13 +110,13 @@ export const rejectedLogins = createQuarantine(); */ export async function fetchLiveStreams(logins, { report } = {}) { const out = new Map(); - const { results, rejected, unresolved } = await fetchIsolated( + const { results, rejected, unresolved, inconclusive } = await fetchIsolated( logins, 100, (batch) => helix('/streams', { user_login: batch }), { quarantine: rejectedLogins } ); - report?.({ rejected, unresolved }); + report?.({ rejected, unresolved, inconclusive }); for (const s of results) out.set(s.user_login.toLowerCase(), s); return out; } @@ -205,8 +205,14 @@ export async function tick() { let unresolved = new Set(); try { streams = await fetchLiveStreams(logins, { - report: ({ rejected, unresolved: left }) => { + report: ({ rejected, unresolved: left, inconclusive }) => { unresolved = left; + if (inconclusive) { + log.warn( + 'twitch-alerts', + 'Every request of this poll was refused and no earlier accepted value is known to check against, so nothing was quarantined.' + ); + } const warning = rejectedWarning('Twitch logins', rejected, (login) => [ ...new Set(jobs.filter((j) => j.alert.login === login).map((j) => j.guildId)), ]); diff --git a/test/liveAlertsIsolation.test.js b/test/liveAlertsIsolation.test.js index dff3d732..bb711042 100644 --- a/test/liveAlertsIsolation.test.js +++ b/test/liveAlertsIsolation.test.js @@ -80,9 +80,10 @@ test('fetchIsolated: a failure that is not about a value still fails the whole p } }); -test('fetchIsolated: when no request of the poll succeeded the request is at fault, nothing is quarantined', async () => { +test('fetchIsolated: when nothing succeeds and a value known to be good is refused too, the request is at fault', async () => { for (const count of [3, 40]) { const quarantine = createQuarantine(); + quarantine.setGood('known-good'); // accepted in an earlier poll const items = Array.from({ length: count }, (_, i) => `item${i}`); await assert.rejects( fetchIsolated(items, 50, async () => Promise.reject(rejected(400)), { quarantine }), @@ -92,6 +93,101 @@ test('fetchIsolated: when no request of the poll succeeded the request is at fau } }); +test('fetchIsolated: nothing succeeds but the known-good value is accepted: the values are at fault', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const out = await fetchIsolated( + ['bad1', 'bad2'], + 50, + async (batch) => { + if (batch.every((i) => i === 'known-good')) return batch; + throw rejected(400); + }, + { quarantine } + ); + assert.deepEqual(out.rejected.map((r) => r.item).sort(), ['bad1', 'bad2']); + assert.equal(quarantine.size, 2); + assert.equal(out.inconclusive, false); +}); + +test('fetchIsolated: nothing succeeds and nothing is known to be good: nothing is quarantined, and it says so', async () => { + const quarantine = createQuarantine(); + const out = await fetchIsolated(['a', 'b', 'c'], 50, async () => Promise.reject(rejected(400)), { + quarantine, + }); + assert.equal(out.inconclusive, true); + assert.equal(quarantine.size, 0); + assert.deepEqual([...out.unresolved].sort(), ['a', 'b', 'c']); +}); + +test('fetchIsolated: when the platform flaps (some answer, some are refused) and the canary is refused, nothing is quarantined', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const out = await fetchIsolated( + ['fine1', 'flaky', 'fine2'], + 1, + async (batch) => { + if (batch[0] === 'flaky' || batch[0] === 'known-good') throw rejected(404); + return batch; + }, + { quarantine } + ); + assert.deepEqual(out.results.sort(), ['fine1', 'fine2']); + assert.equal(quarantine.size, 0, 'a value the platform refuses only while it flaps is not quarantined'); + assert.ok(out.unresolved.has('flaky'), 'its state is left alone this poll'); + assert.equal(quarantine.lastGood, 'fine2', 'a fresh good value for the next canary'); +}); + +test('fetchIsolated: poison values filling the first batch cannot fail the poll or leave later batches out', async () => { + const quarantine = createQuarantine(); + const poison = Array.from({ length: 100 }, (_, i) => `_bad${i}`); + const victims = Array.from({ length: 30 }, (_, i) => `victim${i}`); + const ask = async (batch) => { + if (batch.some((i) => i.startsWith('_'))) throw rejected(400); + return batch; + }; + // Bisecting 100 poison values takes about 199 requests; the budget is 120. + const first = await fetchIsolated([...poison, ...victims], 100, ask, { quarantine }); + assert.deepEqual([...first.results].sort(), [...victims].sort(), 'the later batch was fetched'); + assert.ok(first.rejected.length > 0 && first.rejected.length < 100, 'some found, the budget ended'); + // Later polls finish the job, with what was found left out. + let polls = 1; + while (polls < 6 && poison.some((p) => !quarantine.has(p))) { + const next = await fetchIsolated([...poison, ...victims], 100, ask, { quarantine }); + assert.ok(next.results.length > 0 || next.unresolved.size > 0); + polls += 1; + } + assert.ok( + poison.every((p) => quarantine.has(p)), + `all quarantined within ${polls} polls` + ); + assert.ok(polls <= 4, `bounded: ${polls} polls`); + const last = await fetchIsolated([...poison, ...victims], 100, ask, { quarantine }); + assert.deepEqual( + [...last.results].sort(), + [...victims].sort(), + 'once the poison is quarantined, a clean poll' + ); +}); + +test('fetchIsolated: the start rotates, so the same values are not always last', async () => { + const quarantine = createQuarantine(); + const items = Array.from({ length: 8 }, (_, i) => `i${i}`); + const firsts = []; + for (let n = 0; n < 4; n += 1) { + await fetchIsolated( + items, + 2, + async (batch) => { + if (n === firsts.length && !firsts[n]) firsts.push(batch[0]); + return batch; + }, + { quarantine } + ); + } + assert.equal(new Set(firsts).size, 4, 'each poll starts somewhere else'); +}); + test('fetchIsolated: a request budget leaves the rest of the poll out instead of failing it', async () => { const quarantine = createQuarantine(); const items = [ @@ -107,7 +203,7 @@ test('fetchIsolated: a request budget leaves the rest of the poll out instead of if (batch.some((i) => i.startsWith('_'))) throw rejected(400); return batch; }, - { quarantine, maxRequests: 6 } + { quarantine, extraRequests: 2 } ); assert.equal(calls, 6); assert.ok(out.results.length > 0, 'what was fetched is kept'); @@ -437,3 +533,62 @@ test('kick tick: a request-level 403 quarantines no good slug', async (t) => { runtime.client = null; } }); + +test('twitch tick: two guilds with 50 poisoned logins each, first in order, cannot block the guilds after them', async (t) => { + twitch.rejectedLogins.setGood(null); // an earlier test may have left a value this one's stub refuses + t.mock.method(log, 'warn', () => {}); + const failed = t.mock.method(log, 'error', () => {}); + const sent = []; + const second = '900000000000000503'; + const guilds = new Map([ + [ATTACKER, fakeGuild(ATTACKER, sent)], + [second, fakeGuild(second, sent)], + [VICTIM, fakeGuild(VICTIM, sent)], + ]); + runtime.client = { isReady: () => true, guilds: { cache: guilds } }; + for (const g of guilds.values()) g.client = runtime.client; + const poisoned = (tag) => + Array.from({ length: 50 }, (_, i) => ({ + login: `_${tag}${String(i).padStart(2, '0')}`, + channelId: CH_A, + })); + await setGuildModule(ATTACKER, 'twitch-alerts', { enabled: true, config: { alerts: poisoned('aa') } }); + await setGuildModule(second, 'twitch-alerts', { enabled: true, config: { alerts: poisoned('bb') } }); + await setGuildModule(VICTIM, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: 'victimlate', channelId: CH_A }] }, + }); + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.twitch.tv') return Response.json({ access_token: 't', expires_in: 3600 }); + const logins = u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login'); + if (logins.some((l) => l.startsWith('_'))) return new Response('{}', { status: 400 }); + if (u.pathname.endsWith('/streams')) { + return Response.json({ data: logins.filter((l) => l === 'victimlate').map(twitchStream) }); + } + return Response.json({ data: [] }); + }; + try { + // The batches are cut from a rotating start, so a guild behind the poisoned ones may share a + // batch with them in a poll; the poll still does not fail, and within a few polls the poisoned + // logins are quarantined and the guild is announced. + const all = [...poisoned('aa'), ...poisoned('bb')]; + let polls = 0; + while (polls < 8 && (sent.length === 0 || all.some((p) => !twitch.rejectedLogins.has(p.login)))) { + await twitch.tick(); + polls += 1; + assert.equal(failed.mock.callCount(), 0, `poll ${polls} did not fail`); + } + assert.equal(sent.length, 1, `the guild after the poisoned ones is announced (after ${polls} polls)`); + assert.equal(sent[0].guild, VICTIM); + assert.ok( + all.every((p) => twitch.rejectedLogins.has(p.login)), + 'all poisoned logins quarantined' + ); + assert.ok(polls <= 6, `bounded: ${polls} polls`); + } finally { + globalThis.fetch = realFetch; + runtime.client = null; + } +}); From 8f0c0f5f421e8b813eda4a6a61de719f0312b23a Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 02:20:06 +0100 Subject: [PATCH 13/21] fix(tickets): cap the open ticket checks and make the lookup budget fairer - The open ticket step honours the remembered not-a-member answers, has its own cap (5 fetches per DM, 10 per user per minute, separate from the lookup budget), and remembers a member who left such a server for ten minutes. Past the cap the open list is marked incomplete, which makes the handler ask again. A menu pick honours the same memory and the same allowance, so clicking the menu over and over costs one fetch. - The last 30 of the shared 120 fetches a minute are kept for users who have used fewer than 4 of their own. - Not a member is remembered for three minutes again (a member join clears it where Sylo sees the join); only a refused lookup (403 / 50001) is remembered for ten. --- docs/modules/tickets.md | 17 ++-- src/bot/events/dmTickets.js | 6 +- src/modules/tickets.js | 110 +++++++++++++++++++----- test/ticketGuildLookup.test.js | 147 ++++++++++++++++++++++++++++++++- 4 files changed, 249 insertions(+), 31 deletions(-) diff --git a/docs/modules/tickets.md b/docs/modules/tickets.md index 4a83ac90..ce9683df 100644 --- a/docs/modules/tickets.md +++ b/docs/modules/tickets.md @@ -35,13 +35,16 @@ empty) and closes it. The member can open a new one by DMing again. To find the servers a member shares with the bot, Sylo first looks at the servers where the member has an open ticket (from its own database, so this survives a restart): a follow-up message always -goes to the open ticket. Then it uses its member cache: a server whose full member list is cached -(Server Members intent on) answers without any request to Discord. For the other servers it asks -Discord for the one member: at most 8 servers per message, 16 requests a minute per member and 120 a -minute for the whole bot, and it remembers "not a member" (or a refused lookup) for ten minutes. -Without the Server Members intent a member is still found this way. If a message could not check -every server yet, Sylo never opens a new ticket from it: it asks the member to send the message again -a few seconds later, which continues with the servers not checked yet. +goes to the open ticket. Those servers are checked with a few requests per message (5, and 10 a +minute per member); a member who left such a server is remembered as gone for ten minutes. Then it +uses its member cache: a server whose full member list is cached (Server Members intent on) +answers without any request to Discord. For the other servers it asks Discord for the one member: +at most 8 servers per message, 16 requests a minute per member and 120 a minute for the whole bot +(the last 30 are kept for members who have used little), and it remembers "not a member" for three +minutes (a join clears it at once where Sylo sees the join). A server that refuses the lookup is +skipped for ten minutes. Without the Server Members intent a member is still found this way. If a +message could not check every server yet, Sylo never opens a new ticket from it: it asks the member +to send the message again a few seconds later, which continues with the servers not checked yet. `/forget` removes the member's ticket history: every message in their tickets, staff replies included. Replies already delivered by DM are not clawed back. diff --git a/src/bot/events/dmTickets.js b/src/bot/events/dmTickets.js index b664ef0a..30b4752d 100644 --- a/src/bot/events/dmTickets.js +++ b/src/bot/events/dmTickets.js @@ -5,7 +5,7 @@ import { guildKey, guildByKey, platformOfClient } from '../../platform/platformOf.js'; import { config } from '../../config.js'; import { Events, StringSelectMenuBuilder, ActionRowBuilder } from 'discord.js'; -import { lookupTicketGuilds, canOpenTicketIn, ingestUserDM } from '../../modules/tickets.js'; +import { lookupTicketGuilds, canOpenTicketIn, forgetNotMember, ingestUserDM } from '../../modules/tickets.js'; import { registerComponent } from '../lib/components.js'; import { handleQuestionnaireMessage, looksLikeQuestionnairePrompt } from '../lib/dmQuestionnaire.js'; import { lateAnswerWindowMs } from '../../modules/applications.js'; @@ -159,6 +159,10 @@ registerComponent('tickets', SELECT_ID, handleSelect); /** @param {import('discord.js').Client} client */ export function register(client, { isCommand = () => false } = {}) { + // A person who joins a server is no longer "not a member" there (Discord, with the intent on). + client.on(Events.GuildMemberAdd, (member) => { + if (member?.guild) forgetNotMember(member.id, guildKey(member.guild)); + }); client.on(Events.MessageCreate, async (message) => { try { if (message.guild || message.author?.bot) return; diff --git a/src/modules/tickets.js b/src/modules/tickets.js index 97198133..bd288eca 100644 --- a/src/modules/tickets.js +++ b/src/modules/tickets.js @@ -176,8 +176,23 @@ export const MEMBER_PROBES_PER_DM = 8; export const MEMBER_PROBES_PER_USER_PER_MINUTE = 16; /** Most REST member fetches all users together may cause per minute (open tickets are not counted). */ export const MEMBER_PROBES_PER_MINUTE = 120; -/** A "not a member" answer, or a refused fetch, is remembered this long. */ -export const NOT_MEMBER_TTL_MS = 10 * 60_000; +/** + * The last part of the shared minute is kept for users who have used little of it, so a few users + * cannot leave everyone else with "send again": above SHARED_OPEN_LIMIT a user may only take + * fetches while they have used fewer than STARTER_PROBES of their own. + */ +export const SHARED_OPEN_LIMIT = 90; +export const STARTER_PROBES = 4; +/** Most fetches for the servers where the member has an open ticket: per DM, and per user per minute. */ +export const OPEN_PROBES_PER_DM = 5; +export const OPEN_PROBES_PER_USER_PER_MINUTE = 10; +/** A "not a member" answer is remembered this long (a join event clears it where Sylo gets one). */ +export const NOT_MEMBER_TTL_MS = 3 * 60_000; +/** + * A refused fetch (403 / 50001: the bot may not look in that server) and a member who left a + * server where they had an open ticket are remembered this long. + */ +export const LONG_MEMORY_TTL_MS = 10 * 60_000; /** A complete answer for one sender is reused this long (a per-user cooldown on the lookup). */ export const LOOKUP_REUSE_MS = 10_000; const HINT_TTL_MS = 24 * 60 * 60_000; @@ -190,14 +205,17 @@ const checkedAt = new Map(); const foundBefore = new Map(); /** `${platform}|${userId}` -> { at, result } (result is a promise while the lookup runs). */ const lookups = new Map(); -/** userId -> { start, count }: the user's fetches in the current minute. */ +/** userId -> { start, count }: the user's fetches in the current minute (shared budget). */ const userWindows = new Map(); +/** userId -> { start, count }: the user's fetches for open tickets and menu picks in the current minute. */ +const openWindows = new Map(); let probeWindow = { start: 0, count: 0 }; sweepMap(notMember, (until, now) => until <= now); sweepMap(checkedAt, (at, now) => now - at > HINT_TTL_MS); sweepMap(foundBefore, (h, now) => now - h.at > HINT_TTL_MS); sweepMap(lookups, (l, now) => now - l.at > LOOKUP_REUSE_MS); sweepMap(userWindows, (w, now) => now - w.start >= 60_000, 60_000); +sweepMap(openWindows, (w, now) => now - w.start >= 60_000, 60_000); /** Test hook: forget every remembered answer and the probe budgets. */ export function _resetTicketLookup() { @@ -206,6 +224,7 @@ export function _resetTicketLookup() { checkedAt.clear(); lookups.clear(); userWindows.clear(); + openWindows.clear(); probeWindow = { start: 0, count: 0 }; } @@ -220,41 +239,68 @@ function memberCacheComplete(guild) { return Number.isFinite(guild.memberCount) && (guild.members.cache?.size ?? 0) >= guild.memberCount; } -/** Take one REST member fetch from the user's and the bot's per-minute budgets; false when either is used up. */ -function takeProbe(userId) { +/** Count one more in a user's minute window; false (and no count) when `limit` is reached. */ +function takeWindow(map, userId, limit) { const now = Date.now(); - if (now - probeWindow.start >= 60_000) probeWindow = { start: now, count: 0 }; - let w = userWindows.get(userId); + let w = map.get(userId); if (!w || now - w.start >= 60_000) { w = { start: now, count: 0 }; - userWindows.set(userId, w); + map.set(userId, w); } - if (probeWindow.count >= MEMBER_PROBES_PER_MINUTE || w.count >= MEMBER_PROBES_PER_USER_PER_MINUTE) { - return false; - } - probeWindow.count++; + if (w.count >= limit) return false; w.count++; return true; } +/** A member of a server joined: forget that they were found not to be one (Discord, intent on). */ +export function forgetNotMember(userId, key) { + notMember.delete(`${userId}|${key}`); +} + +/** + * Take one REST member fetch from the user's and the bot's per-minute budgets; false when either + * is used up, or when the shared minute is nearly used and this user already had their share. + */ +function takeProbe(userId) { + const now = Date.now(); + if (now - probeWindow.start >= 60_000) probeWindow = { start: now, count: 0 }; + const used = userWindows.get(userId); + const mine = used && now - used.start < 60_000 ? used.count : 0; + if (probeWindow.count >= MEMBER_PROBES_PER_MINUTE) return false; + if (probeWindow.count >= SHARED_OPEN_LIMIT && mine >= STARTER_PROBES) return false; + if (!takeWindow(userWindows, userId, MEMBER_PROBES_PER_USER_PER_MINUTE)) return false; + probeWindow.count++; + return true; +} + /** * One REST member fetch. + * @param {{ ttl?: number }} [o] how long "not a member" is remembered (a refusal is always + * remembered for LONG_MEMORY_TTL_MS) * @returns {Promise} true member; false not a member, or the platform refuses to - * say (remembered for NOT_MEMBER_TTL_MS, so it is not asked again at once); null unknown (a rate - * limit or a network error: not remembered) + * say (remembered, so it is not asked again at once); null unknown (a rate limit or a network + * error: not remembered) */ -async function probeMember(guild, userId) { - const remember = () => notMember.set(`${userId}|${guildKey(guild)}`, Date.now() + NOT_MEMBER_TTL_MS); +async function probeMember(guild, userId, { ttl = NOT_MEMBER_TTL_MS } = {}) { + const remember = (ms) => { + const key = `${userId}|${guildKey(guild)}`; + notMember.set(key, Math.max(notMember.get(key) ?? 0, Date.now() + ms)); + }; checkedAt.set(`${userId}|${guildKey(guild)}`, Date.now()); try { // A platform may answer "no such member" with null instead of an error. if (await guild.members.fetch(userId)) return true; - remember(); + remember(ttl); return false; } catch (err) { - // 10007 / 10013: unknown member / user. 403 / 50001: the bot may not look in this server. - if ([10007, 10013, 50001].includes(err?.code) || [403, 404].includes(err?.status)) { - remember(); + // 403 / 50001: the bot may not look in this server: skip it for a long while. + if (err?.code === 50001 || err?.status === 403) { + remember(LONG_MEMORY_TTL_MS); + return false; + } + // 10007 / 10013: unknown member / user. + if ([10007, 10013].includes(err?.code) || err?.status === 404) { + remember(ttl); return false; } return null; @@ -270,13 +316,28 @@ async function findTicketGuilds(user, client) { // 1. Open tickets, from the database: complete, and kept across restarts. let openComplete = true; + let openProbes = 0; const handled = new Set(); for (const key of await openTicketGuildKeys(user.id)) { const guild = byKey.get(key); if (!guild) continue; // another platform, or Sylo left the server handled.add(key); if (!(await isModuleEnabled(key, 'tickets'))) continue; - const isMember = cachedMember(guild, user.id) ? true : await probeMember(guild, user.id); + let isMember = cachedMember(guild, user.id) ? true : null; + if (isMember === null) { + if ((notMember.get(`${user.id}|${key}`) ?? 0) > Date.now()) { + isMember = false; // left, or refused: remembered + } else if ( + openProbes >= OPEN_PROBES_PER_DM || + !takeWindow(openWindows, user.id, OPEN_PROBES_PER_USER_PER_MINUTE) + ) { + openComplete = false; // not checked this time: the DM handler asks again or shows the picker + } else { + openProbes++; + // A member with an open ticket who is gone has most likely left: remember it for long. + isMember = await probeMember(guild, user.id, { ttl: LONG_MEMORY_TTL_MS }); + } + } if (isMember) { found.push(guild); openGuilds.push(guild); @@ -356,7 +417,7 @@ export function lookupTicketGuilds(user, client) { /** * Can this user open a ticket in this one server (tickets on, user a member)? Cheap: the member * cache, or the DM lookup that just ran, or one fetch. The server was one the bot offered in a - * menu a moment ago, so that one fetch is not counted against the budgets. + * menu a moment ago, so that one fetch is not counted against the shared budget. */ export async function canOpenTicketIn(user, guild) { if (!(await isModuleEnabled(guildKey(guild), 'tickets'))) return false; @@ -367,6 +428,11 @@ export async function canOpenTicketIn(user, guild) { if (guilds.some((g) => guildKey(g) === guildKey(guild))) return true; } if (memberCacheComplete(guild)) return false; + // A server the member left (or that refuses lookups) stays refused for a while, and a pick costs + // at most one fetch from the user's own small allowance: clicking the menu again and again is + // bounded. + if ((notMember.get(`${user.id}|${guildKey(guild)}`) ?? 0) > Date.now()) return false; + if (!takeWindow(openWindows, user.id, OPEN_PROBES_PER_USER_PER_MINUTE)) return false; return (await probeMember(guild, user.id)) === true; } diff --git a/test/ticketGuildLookup.test.js b/test/ticketGuildLookup.test.js index 31b25fd5..5b50de46 100644 --- a/test/ticketGuildLookup.test.js +++ b/test/ticketGuildLookup.test.js @@ -15,6 +15,11 @@ import { MEMBER_PROBES_PER_USER_PER_MINUTE, LOOKUP_REUSE_MS, NOT_MEMBER_TTL_MS, + LONG_MEMORY_TTL_MS, + OPEN_PROBES_PER_DM, + OPEN_PROBES_PER_USER_PER_MINUTE, + SHARED_OPEN_LIMIT, + STARTER_PROBES, } from '../src/modules/tickets.js'; import { register } from '../src/bot/events/dmTickets.js'; import { routeComponent } from '../src/bot/lib/components.js'; @@ -106,7 +111,10 @@ test('incomplete caches: fetches are capped per DM and per user, remembered, and assert.equal(third.incomplete, true); t.mock.timers.tick(61_000); - for (let n = 0; n < 6 && (await lookupTicketGuilds(SENDER, client)).incomplete; n++) { + // Two messages a minute, as a member who retries would send them. + for (let n = 0; n < 6; n++) { + await lookupTicketGuilds(SENDER, client); + if (!(await lookupTicketGuilds(SENDER, client)).incomplete) break; t.mock.timers.tick(61_000); } assert.equal(fetches.length, 40, 'each server was fetched exactly once'); @@ -396,3 +404,140 @@ test('the DM handler asks the sender to try again when the lookup was cut short, assert.equal(replies.length, 2); assert.match(replies[1], /not set up/, 'all servers checked by the second DM: a definite answer'); }); + +// --- round 3: left servers, fairness, how long "not a member" is remembered --------------------- + +test('a member with open tickets in servers they left costs at most one fetch per server per 10 minutes', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 20_000_000 }); + const user = { id: '750000000000000001' }; + const fetches = []; + const guilds = []; + for (let i = 0; i < 20; i++) { + const g = await ticketGuild(fetches); // the member left: every fetch says unknown member + await createTicket(g.id, user.id); + guilds.push(g); + } + const client = clientOf(guilds); + // A DM every 11 seconds for 9 minutes. + for (let n = 0; n < 49; n++) { + await lookupTicketGuilds(user, client); + t.mock.timers.tick(11_000); + } + assert.ok(fetches.length <= 20, `${fetches.length} fetches for 20 servers in 9 minutes`); + assert.equal(new Set(fetches.map((f) => f.guild)).size, fetches.length, 'no server asked twice'); + const settled = await lookupTicketGuilds(user, client); + assert.deepEqual([settled.openGuilds.length, settled.openComplete], [0, true]); + // After the memory runs out each server is asked again, still within the caps per DM and minute. + t.mock.timers.tick(11 * 60_000); + fetches.length = 0; + const after = await lookupTicketGuilds(user, client); + assert.ok(fetches.length <= 5, 'at most 5 open-ticket fetches in one DM'); + assert.equal(after.openComplete, false, 'not all checked yet: the DM handler asks again'); +}); + +test('the open ticket checks have a cap per DM and a small allowance per user and minute', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 21_000_000 }); + const user = { id: '750000000000000002' }; + const fetches = []; + const guilds = []; + for (let i = 0; i < 30; i++) { + const g = await ticketGuild(fetches, { members: [user.id] }); + await createTicket(g.id, user.id); + guilds.push(g); + } + const client = clientOf(guilds); + const first = await lookupTicketGuilds(user, client); + assert.equal(fetches.length, OPEN_PROBES_PER_DM); + assert.equal(first.openComplete, false); + await lookupTicketGuilds(user, client); + await lookupTicketGuilds(user, client); + assert.equal(fetches.length, OPEN_PROBES_PER_USER_PER_MINUTE, 'the user allowance for open tickets'); + // It does not touch the user budget for new-ticket lookups, nor the shared one. + const extra = await ticketGuild(fetches); + const before = fetches.length; + await lookupTicketGuilds(user, clientOf([extra])); + assert.equal(fetches.length, before + 1, 'a new-ticket lookup is not blocked by the open ticket allowance'); +}); + +test('a repeated menu pick for a server the member left costs one fetch', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 22_000_000 }); + const user = { id: '750000000000000003' }; + const fetches = []; + const left = await ticketGuild(fetches); + for (let n = 0; n < 30; n++) assert.equal(await canOpenTicketIn(user, left), false); + assert.equal(fetches.length, 1); +}); + +test('the shared budget keeps a share for users who have used little of it', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 23_000_000 }); + const fetches = []; + const guilds = []; + for (let i = 0; i < 60; i++) guilds.push(await ticketGuild(fetches)); + const client = clientOf(guilds); + // Eight busy accounts use their whole minute each. + for (let u = 0; u < 8; u++) { + const busy = { id: `76000000000000000${u}` }; + for (let n = 0; n < 3; n++) await lookupTicketGuilds(busy, client); + } + assert.ok( + fetches.length <= SHARED_OPEN_LIMIT + STARTER_PROBES * 2, + `the busy ones stopped at ${fetches.length}` + ); + assert.ok( + fetches.length < 8 * MEMBER_PROBES_PER_USER_PER_MINUTE, + 'not all of their allowance went through' + ); + const fresh = { id: '770000000000000000' }; + const r = await lookupTicketGuilds(fresh, client); + assert.equal( + fetches.filter((f) => f.id === fresh.id).length, + STARTER_PROBES, + 'a new user still gets a start' + ); + assert.equal(r.incomplete, true); +}); + +test('"not a member" is remembered shortly, a refused lookup for long, and a join clears it', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 24_000_000 }); + const user = { id: '750000000000000004' }; + const fetches = []; + const plain = await ticketGuild(fetches); + const refusing = await ticketGuild(fetches, { refuse: true }); + const client = clientOf([plain, refusing]); + await lookupTicketGuilds(user, client); + assert.equal(fetches.length, 2); + + t.mock.timers.tick(NOT_MEMBER_TTL_MS + 1000); + await lookupTicketGuilds(user, client); + assert.deepEqual( + fetches.slice(2).map((f) => f.guild), + [plain.id], + 'the plain one is asked again, the refusing one is not yet' + ); + + t.mock.timers.tick(LONG_MEMORY_TTL_MS); + await lookupTicketGuilds(user, client); + assert.ok( + fetches.slice(3).some((f) => f.guild === refusing.id), + 'the refusal expires too' + ); + + // A join event clears the memory at once. + fetches.length = 0; + await lookupTicketGuilds(user, client); + assert.equal(fetches.length, 0, 'remembered'); + const events = new EventEmitter(); + register(events); + events.emit('guildMemberAdd', { id: user.id, guild: plain }); + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + await lookupTicketGuilds(user, client); + assert.deepEqual( + fetches.map((f) => f.guild), + [plain.id] + ); +}); From c7a0bcb5030380c764edbb1c6416dd213cdea27a Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 02:20:06 +0100 Subject: [PATCH 14/21] fix(backup): link the docs on the default branch, test the Postgres restore rules through the route The role warning links docs/postgres.md on the default branch, which has the section from the merge on, instead of a release tag that may not. A Postgres-only route test checks the confirmation for an uploaded dump: every missing or wrong field is a 400, and with both fields the request reaches the dump check without restoring anything. --- src/db/dbRole.js | 8 ++- test/backupRestoreRoute.postgres.test.js | 77 ++++++++++++++++++++++++ test/dbRole.test.js | 7 +-- 3 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 test/backupRestoreRoute.postgres.test.js diff --git a/src/db/dbRole.js b/src/db/dbRole.js index c794963d..d36c3073 100644 --- a/src/db/dbRole.js +++ b/src/db/dbRole.js @@ -9,10 +9,12 @@ // should have none of that: docs/postgres.md (Least-privilege database role). import { config } from '../config.js'; import { prepare } from './driver.js'; -import { BUILD, REPO_URL } from '../bot/lib/buildInfo.js'; +import { REPO_URL } from '../bot/lib/buildInfo.js'; -// The docs of this version, so the link matches the behaviour the reader runs. -export const ROLE_DOCS_URL = `${REPO_URL}/blob/v${BUILD.version}/docs/postgres.md#least-privilege-database-role`; +// The docs on the default branch: a link to a release tag would point at docs without this +// section until the first release that has it (and 404 for a build that has no tag). The section +// only ever gets better, so the newest text is the right one for every version. +export const ROLE_DOCS_URL = `${REPO_URL}/blob/main/docs/postgres.md#least-privilege-database-role`; /** * @typedef {object} RolePrivileges diff --git a/test/backupRestoreRoute.postgres.test.js b/test/backupRestoreRoute.postgres.test.js new file mode 100644 index 00000000..ede6fc1a --- /dev/null +++ b/test/backupRestoreRoute.postgres.test.js @@ -0,0 +1,77 @@ +// The restore route on a Postgres deployment: an uploaded dump needs its name and the +// acknowledgement that its SQL runs with the database role's rights. Skipped without DATABASE_URL. +// config.js reads DATABASE_URL once at import, so this is the only place the Postgres rules can be +// tested through the route. Nothing is restored: with both fields the request gets past the +// confirmation and stops at the dump check (the file is not a dump), and a real restore would +// end the process. +import './helpers/isolateSqlite.js'; +import './helpers/openMode.js'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const url = process.env.DATABASE_URL; + +test( + 'restoring an uploaded dump through the route needs confirm and acknowledgeSql on Postgres', + { skip: !url && 'DATABASE_URL not set (sqlite-only run)' }, + async (t) => { + const { runtime } = await import('../src/runtime.js'); + const { createApp } = await import('../src/web/server.js'); + const { fakeGuild, makeSink } = await import('./helpers/fakeGuild.js'); + const { backupDir } = await import('../src/db/backup.js'); + const { closePostgres } = await import('../src/db/driver.js'); + + runtime.client = fakeGuild({ sink: makeSink() }).client; + const server = await new Promise((resolve) => { + const s = createApp().listen(0, () => resolve(s)); + }); + const imported = 'sylo-imported-2026-11-16-10-00-00-000.dump'; + const own = 'sylo-manual-2026-11-16-10-00-00-001.dump'; + const files = [join(backupDir(), imported), join(backupDir(), own)]; + for (const f of files) writeFileSync(f, 'PGDMP not a real dump'); + t.after(async () => { + for (const f of files) rmSync(f, { force: true }); + server.closeAllConnections?.(); + server.close(); + runtime.client = null; + await closePostgres(); + }); + + const base = `http://127.0.0.1:${server.address().port}/api/v2/health/backups`; + const restore = (name, body) => + fetch(`${base}/${name}/restore`, { + method: 'POST', + headers: { accept: 'application/json', 'content-type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const error = async (res) => (await res.json()).error; + + for (const body of [undefined, {}, { acknowledgeSql: true }]) { + const res = await restore(imported, body); + assert.equal(res.status, 400, JSON.stringify(body)); + assert.match(await error(res), /needs \{ "confirm"/); + } + for (const ack of [undefined, false, 'true', 1]) { + const res = await restore(imported, { confirm: imported, acknowledgeSql: ack }); + assert.equal(res.status, 400, String(ack)); + assert.match(await error(res), /acknowledgeSql/); + } + const wrong = await restore(imported, { confirm: own, acknowledgeSql: true }); + assert.equal(wrong.status, 400); + assert.match(await error(wrong), /name of the snapshot/); + + // Both fields: past the confirmation, stopped by the dump check (it is not a dump). + const both = await restore(imported, { confirm: imported, acknowledgeSql: true }); + assert.equal(both.status, 400); + const message = await error(both); + assert.doesNotMatch(message, /acknowledgeSql|confirm/); + assert.ok(existsSync(files[0]), 'nothing was restored or removed'); + + // A snapshot Sylo made itself needs no body (the app sends none): also stopped by the dump check. + const plain = await restore(own); + assert.equal(plain.status, 400); + assert.doesNotMatch(await error(plain), /acknowledgeSql|confirm/); + } +); diff --git a/test/dbRole.test.js b/test/dbRole.test.js index 17232339..5bc9c5aa 100644 --- a/test/dbRole.test.js +++ b/test/dbRole.test.js @@ -46,11 +46,8 @@ test('superuser, CREATEROLE and CREATEDB each warn, naming the right and linking assert.deepEqual(roleRisks({ ...plain, superuser: true, createDb: true }), ['SUPERUSER', 'CREATEDB']); }); -test('the docs link points at the least-privilege section of the docs of this version', () => { - assert.match( - ROLE_DOCS_URL, - /\/blob\/v\d+\.\d+\.\d+[^/]*\/docs\/postgres\.md#least-privilege-database-role$/ - ); +test('the docs link points at the least-privilege section of docs/postgres.md on the default branch', () => { + assert.match(ROLE_DOCS_URL, /\/blob\/main\/docs\/postgres\.md#least-privilege-database-role$/); }); test('on SQLite there is nothing to read and nothing to warn about', async () => { From dcfc4493eddad71f9c12c67a6dbc2a9596848712 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 02:41:00 +0100 Subject: [PATCH 15/21] fix(live-alerts): search failed batches depth first so every poll makes progress Breadth-first halving spent the whole budget on the first levels when several batches were full of rejected values, so no value was ever isolated, nothing was quarantined and the cost stayed high for good. The search is now depth first: one failed batch is searched down to single values before the next is started, so each poll quarantines about 60 dense or 8 sparse values. Chunks left unchecked when the budget ends are remembered as suspects and batched apart from the clean values in the next poll, so the clean ones are always fetched. One warning per poll, with counts and no values, says when the search ran out. A 429 or 5xx on the canary request counts as a refused canary: the poll keeps what it fetched and only skips the quarantine. --- src/modules/kickAlerts.js | 23 +++- src/modules/lib/batchIsolation.js | 76 ++++++++--- src/modules/twitchAlerts.js | 14 +- test/liveAlertsIsolation.test.js | 207 ++++++++++++++++++++++++++++++ 4 files changed, 289 insertions(+), 31 deletions(-) diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index ee109955..a7e5506a 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -111,15 +111,20 @@ export const rejectedSlugs = createQuarantine(); /** * Live channels for a set of slugs, keyed by lowercased slug. Only currently-live * channels are included. A slug that Kick rejects is isolated (see fetchIsolated) and left out. - * `report({ rejected, unresolved, inconclusive })` is called once with the slugs rejected now and every slug + * `report({ rejected, unresolved, inconclusive, skipped })` is called once with the slugs rejected now and every slug * this poll knows nothing about. */ export async function fetchLiveChannels(slugs, { report } = {}) { const out = new Map(); - const { results, rejected, unresolved, inconclusive } = await fetchIsolated(slugs, 50, getChannels, { - quarantine: rejectedSlugs, - }); - report?.({ rejected, unresolved, inconclusive }); + const { results, rejected, unresolved, inconclusive, skipped } = await fetchIsolated( + slugs, + 50, + getChannels, + { + quarantine: rejectedSlugs, + } + ); + report?.({ rejected, unresolved, inconclusive, skipped }); for (const ch of results) { if (ch?.stream?.is_live && ch.slug) out.set(String(ch.slug).toLowerCase(), ch); } @@ -195,8 +200,14 @@ export async function tick() { let unresolved = new Set(); try { live = await fetchLiveChannels(slugs, { - report: ({ rejected, unresolved: left, inconclusive }) => { + report: ({ rejected, unresolved: left, inconclusive, skipped }) => { unresolved = left; + if (skipped > 0) { + log.warn( + 'kick-alerts', + `${skipped} saved slugs could not be checked this poll: the platform refuses requests that hold some of them and the search ran out of requests. They are checked again at the next poll, apart from the others.` + ); + } if (inconclusive) { log.warn( 'kick-alerts', diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js index bb059f34..16703fe9 100644 --- a/src/modules/lib/batchIsolation.js +++ b/src/modules/lib/batchIsolation.js @@ -7,10 +7,12 @@ // itself is fine. This always fits the budget, so a clean batch is never left out because // another one needed a long search. // 2. only the batches that failed are halved until the rejected items stand alone. This has its -// own budget (EXTRA_REQUESTS), spread over the failing batches in turn. When it runs out, the -// items of the halves not reached are left out of this poll (never failed): the next poll, -// with the found values already quarantined and the batches cut differently (the start -// rotates), reaches them. A poll never fails because the budget ran out. +// own budget (EXTRA_REQUESTS) and goes depth first: one failed batch is searched down to single +// values before the next is started, so every poll isolates (and so quarantines) values, however +// many there are. When the budget runs out, the items of the chunks not reached are left out of +// this poll (never failed) and remembered as suspects: the next poll batches them apart from +// the clean values, so the clean ones are always fetched, and searches them first. A poll never +// fails because the budget ran out. // The values found are quarantined with a growing backoff after the poll. // // Item fault or request fault? Only the statuses that can mean "this value" count (400, 404, 422); @@ -44,6 +46,8 @@ export function createQuarantine({ const entries = new Map(); let good = null; let turn = 0; + /** @type {Set} */ + let suspects = new Set(); return { /** True while the item is waiting out its backoff. */ has(key) { @@ -79,6 +83,13 @@ export function createQuarantine({ setGood(item) { good = item; }, + /** Values left unchecked inside a failed chunk when the last poll ran out of budget. */ + get suspects() { + return suspects; + }, + setSuspects(items) { + suspects = new Set(items); + }, /** Where the next poll starts in its list of `length` items: a different place each time. */ nextStart(length, step) { const start = length > 0 ? turn % length : 0; @@ -101,7 +112,8 @@ export function createQuarantine({ * rejected: Array<{ item: string, err: Error, waitMs: number, strikes: number }>, * unresolved: Set, * inconclusive: boolean, - * }>} `unresolved`: the items this poll knows nothing about (quarantined before, rejected now, or + * skipped: number, + * }>} `skipped`: how many items were left out only because the budget ran out. `unresolved`: the items this poll knows nothing about (quarantined before, rejected now, or * not reached when the budget ran out); their stored state must be left alone. */ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extraRequests = EXTRA_REQUESTS }) { @@ -134,43 +146,62 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extra unresolved.add(item); return false; }); - const start = quarantine.nextStart(live.length, size); - const order = [...live.slice(start), ...live.slice(0, start)]; + // Values left unchecked in a failed chunk by an earlier poll go in batches of their own, after + // the others, so clean values never wait behind them. + const suspect = live.filter((item) => quarantine.suspects.has(item)); + const clean = live.filter((item) => !quarantine.suspects.has(item)); + const start = quarantine.nextStart(clean.length, size); + const batches = []; + for (const group of [[...clean.slice(start), ...clean.slice(0, start)], suspect]) { + for (let i = 0; i < group.length; i += size) batches.push(group.slice(i, i + size)); + } // Phase 1: one request per batch. - const queue = []; - for (let i = 0; i < order.length; i += size) { - const batch = order.slice(i, i + size); + const failed = []; + for (const batch of batches) { const answer = await ask(batch); if (answer === true) continue; if (batch.length === 1) found.push({ item: batch[0], err: answer }); - else queue.push(batch); + else failed.push(batch); } - // Phase 2: halve the failed batches, in turn, until the budget is used up. + // Phase 2: search the failed batches depth first until the budget is used up. Both halves of a + // chunk are asked; the failed ones go on a stack with the first half on top. let spare = extraRequests; - while (queue.length > 0) { - const batch = queue.shift(); - const mid = Math.ceil(batch.length / 2); - for (const half of [batch.slice(0, mid), batch.slice(mid)]) { + const left = new Set(); // unchecked because the budget ran out + const stack = failed.reverse(); + while (stack.length > 0) { + const chunk = stack.pop(); + if (spare <= 0) { + for (const item of chunk) left.add(item); + continue; + } + const mid = Math.ceil(chunk.length / 2); + const halves = [chunk.slice(0, mid), chunk.slice(mid)]; + const next = []; + for (const half of halves) { if (spare <= 0) { - for (const item of half) unresolved.add(item); + for (const item of half) left.add(item); continue; } spare -= 1; const answer = await ask(half); if (answer === true) continue; if (half.length === 1) found.push({ item: half[0], err: answer }); - else queue.push(half); + else next.push(half); } + for (const half of next.reverse()) stack.push(half); } + for (const item of left) unresolved.add(item); let inconclusive = false; let quarantineFound = found.length > 0; if (quarantineFound) { if (canary && !found.some((f) => f.item === canary)) { - // Is the platform refusing everything? One request with a value that was fine before. - const answer = await ask([canary], false); + // Is the platform refusing everything? One request with a value that was fine before. A + // failure of this request of any kind counts as "refused": it must not throw away what the + // poll already fetched. + const answer = await ask([canary], false).catch((err) => err); if (answer !== true) { quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time if (succeeded === 0) throw found[0].err; @@ -183,6 +214,9 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extra } if (lastOk !== null) quarantine.setGood(lastOk); + // What the next poll should batch apart: the unchecked, and the found that were not quarantined. + quarantine.setSuspects([...left, ...(quarantineFound ? [] : found.map((f) => f.item))]); + const rejected = []; for (const { item, err } of found) { unresolved.add(item); @@ -190,7 +224,7 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extra const waitMs = quarantine.add(item); rejected.push({ item, err, waitMs, strikes: quarantine.strikes(item) }); } - return { results, rejected, unresolved, inconclusive }; + return { results, rejected, unresolved, inconclusive, skipped: left.size }; } /** diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 04a5aa4d..4bc5ca4a 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -105,18 +105,18 @@ export const rejectedLogins = createQuarantine(); /** * Live streams for a set of logins, keyed by lowercased login. A login that Helix rejects is - * isolated (see fetchIsolated) and left out. `report({ rejected, unresolved })` is called once with + * isolated (see fetchIsolated) and left out. `report({ rejected, unresolved, inconclusive, skipped })` is called once with * the logins rejected now and every login this poll knows nothing about. */ export async function fetchLiveStreams(logins, { report } = {}) { const out = new Map(); - const { results, rejected, unresolved, inconclusive } = await fetchIsolated( + const { results, rejected, unresolved, inconclusive, skipped } = await fetchIsolated( logins, 100, (batch) => helix('/streams', { user_login: batch }), { quarantine: rejectedLogins } ); - report?.({ rejected, unresolved, inconclusive }); + report?.({ rejected, unresolved, inconclusive, skipped }); for (const s of results) out.set(s.user_login.toLowerCase(), s); return out; } @@ -205,8 +205,14 @@ export async function tick() { let unresolved = new Set(); try { streams = await fetchLiveStreams(logins, { - report: ({ rejected, unresolved: left, inconclusive }) => { + report: ({ rejected, unresolved: left, inconclusive, skipped }) => { unresolved = left; + if (skipped > 0) { + log.warn( + 'twitch-alerts', + `${skipped} saved logins could not be checked this poll: the platform refuses requests that hold some of them and the search ran out of requests. They are checked again at the next poll, apart from the others.` + ); + } if (inconclusive) { log.warn( 'twitch-alerts', diff --git a/test/liveAlertsIsolation.test.js b/test/liveAlertsIsolation.test.js index bb711042..f4ce2cd6 100644 --- a/test/liveAlertsIsolation.test.js +++ b/test/liveAlertsIsolation.test.js @@ -592,3 +592,210 @@ test('twitch tick: two guilds with 50 poisoned logins each, first in order, cann runtime.client = null; } }); + +// --- round 4: enough poisoned values that one poll cannot search them all ------------------------ + +/** Poll until every poisoned value is quarantined; report how many polls it took and how long a victim waited. */ +async function converge({ size, poisonCount, victimsEachSide, maxPolls }) { + const quarantine = createQuarantine(); + const poison = Array.from({ length: poisonCount }, (_, i) => `_p${i}`); + const victims = [ + ...Array.from({ length: victimsEachSide }, (_, i) => `va${i}`), + ...Array.from({ length: victimsEachSide }, (_, i) => `vb${i}`), + ]; + const items = [...victims.slice(0, victimsEachSide), ...poison, ...victims.slice(victimsEachSide)]; + const ask = async (batch) => { + if (batch.some((i) => i.startsWith('_'))) throw rejected(400); + return batch; + }; + const waited = new Map(victims.map((v) => [v, 0])); + let longest = 0; + let polls = 0; + while (polls < maxPolls && poison.some((p) => !quarantine.has(p))) { + const out = await fetchIsolated(items, size, ask, { quarantine }); + polls += 1; + for (const v of victims) { + waited.set(v, out.results.includes(v) ? 0 : waited.get(v) + 1); + longest = Math.max(longest, waited.get(v)); + } + } + const clean = await fetchIsolated(items, size, ask, { quarantine }); + return { polls, longest, clean, victims, poison, quarantine }; +} + +for (const [label, size, poisonCount] of [ + ['200 Twitch logins', 100, 200], + ['250 Kick slugs', 50, 250], +]) { + test(`fetchIsolated: ${label} spread over several guilds are all quarantined within a bounded number of polls`, async () => { + const r = await converge({ size, poisonCount, victimsEachSide: 30, maxPolls: 12 }); + assert.ok( + r.poison.every((p) => r.quarantine.has(p)), + 'all quarantined' + ); + assert.ok(r.polls <= 6, `${r.polls} polls`); + assert.ok(r.longest <= r.polls, `a victim next to them waited ${r.longest} polls`); + assert.deepEqual( + [...r.clean.results].sort(), + [...r.victims].sort(), + 'then every poll fetches the victims' + ); + assert.equal(r.clean.skipped, 0); + }); +} + +test('fetchIsolated: every poll isolates values, however many there are', async () => { + const quarantine = createQuarantine(); + const poison = Array.from({ length: 400 }, (_, i) => `_p${i}`); + const ask = async (batch) => { + if (batch.some((i) => i.startsWith('_'))) throw rejected(400); + return batch; + }; + let before = 0; + for (let n = 0; n < 4; n += 1) { + await fetchIsolated([...poison, 'victim'], 100, ask, { quarantine }); + assert.ok(quarantine.size >= before + 20, `poll ${n + 1} added ${quarantine.size - before}`); + before = quarantine.size; + } +}); + +test('fetchIsolated: a 429 or 5xx on the canary does not throw away what the poll fetched', async () => { + for (const status of [429, 503]) { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const out = await fetchIsolated( + ['fine1', 'bad', 'fine2'], + 1, + async (batch) => { + if (batch[0] === 'bad') throw rejected(400); + if (batch[0] === 'known-good') throw rejected(status); + return batch; + }, + { quarantine } + ); + assert.deepEqual(out.results.sort(), ['fine1', 'fine2'], String(status)); + assert.equal(quarantine.size, 0, 'not quarantined this poll'); + } +}); + +test('twitch tick: 200 poisoned logins in four guilds converge, with warnings that count and name nothing', async (t) => { + const warned = t.mock.method(log, 'warn', () => {}); + const failed = t.mock.method(log, 'error', () => {}); + twitch.rejectedLogins.setGood(null); + const sent = []; + const owners = ['900000000000000511', '900000000000000512', '900000000000000513', '900000000000000514']; + const guilds = new Map(owners.map((id) => [id, fakeGuild(id, sent)])); + guilds.set(VICTIM, fakeGuild(VICTIM, sent)); + runtime.client = { isReady: () => true, guilds: { cache: guilds } }; + for (const g of guilds.values()) g.client = runtime.client; + const names = []; + for (const [n, id] of owners.entries()) { + const alerts = Array.from({ length: 50 }, (_, i) => ({ + login: `_r4${n}x${String(i).padStart(2, '0')}`, + channelId: CH_A, + })); + names.push(...alerts.map((a) => a.login)); + await setGuildModule(id, 'twitch-alerts', { enabled: true, config: { alerts } }); + } + await setGuildModule(VICTIM, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: 'victimr4', channelId: CH_A }] }, + }); + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.twitch.tv') return Response.json({ access_token: 't', expires_in: 3600 }); + const logins = u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login'); + if (logins.some((l) => l.startsWith('_'))) return new Response('{}', { status: 400 }); + if (u.pathname.endsWith('/streams')) { + return Response.json({ data: logins.filter((l) => l === 'victimr4').map(twitchStream) }); + } + return Response.json({ data: [] }); + }; + try { + let polls = 0; + while (polls < 8 && (sent.length === 0 || names.some((l) => !twitch.rejectedLogins.has(l)))) { + await twitch.tick(); + polls += 1; + } + assert.equal(failed.mock.callCount(), 0); + assert.ok(polls <= 7, `${polls} polls`); + assert.equal(sent.length, 1); + assert.ok(names.every((l) => twitch.rejectedLogins.has(l))); + const skippedWarnings = warned.mock.calls + .map((c) => String(c.arguments[1])) + .filter((m) => /could not be checked this poll/.test(m)); + assert.ok(skippedWarnings.length >= 1, 'warned while the search was cut short'); + for (const m of skippedWarnings) assert.doesNotMatch(m, /_r4/, 'counts only, no values'); + } finally { + globalThis.fetch = realFetch; + runtime.client = null; + } +}); + +test('kick tick: 250 poisoned slugs in five guilds converge and a guild behind them is announced', async (t) => { + t.mock.method(log, 'warn', () => {}); + const failed = t.mock.method(log, 'error', () => {}); + kick.rejectedSlugs.setGood(null); + const sent = []; + const owners = [ + '900000000000000521', + '900000000000000522', + '900000000000000523', + '900000000000000524', + '900000000000000525', + ]; + const guilds = new Map(owners.map((id) => [id, fakeGuild(id, sent)])); + guilds.set(VICTIM, fakeGuild(VICTIM, sent)); + runtime.client = { isReady: () => true, guilds: { cache: guilds } }; + for (const g of guilds.values()) g.client = runtime.client; + const names = []; + for (const [n, id] of owners.entries()) { + const alerts = Array.from({ length: 50 }, (_, i) => ({ + slug: `-r4${n}x${String(i).padStart(2, '0')}`, + channelId: CH_A, + })); + names.push(...alerts.map((a) => a.slug)); + await setGuildModule(id, 'kick-alerts', { enabled: true, config: { alerts } }); + } + await setGuildModule(VICTIM, 'kick-alerts', { + enabled: true, + config: { alerts: [{ slug: 'victimkr4', channelId: CH_A }] }, + }); + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.hostname === 'id.kick.com') return Response.json({ access_token: 't', expires_in: 3600 }); + const slugs = u.searchParams.getAll('slug'); + if (slugs.some((s) => s.startsWith('-'))) return new Response('{}', { status: 400 }); + return Response.json({ + data: slugs + .filter((s) => s === 'victimkr4') + .map((slug) => ({ + slug, + stream_title: 'hi', + category: { name: 'g' }, + stream: { + is_live: true, + start_time: '2026-11-16T00:00:00Z', + viewer_count: 2, + url: 'https://kick.com/victimkr4', + }, + })), + }); + }; + try { + let polls = 0; + while (polls < 9 && (sent.length === 0 || names.some((s) => !kick.rejectedSlugs.has(s)))) { + await kick.tick(); + polls += 1; + } + assert.equal(failed.mock.callCount(), 0); + assert.ok(polls <= 8, `${polls} polls`); + assert.equal(sent.length, 1); + assert.ok(names.every((s) => kick.rejectedSlugs.has(s))); + } finally { + globalThis.fetch = realFetch; + runtime.client = null; + } +}); From f4f634a8086dc903537294821929aef9bf4b7ea6 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 02:41:00 +0100 Subject: [PATCH 16/21] fix(tickets): do not open a new ticket elsewhere on the strength of a remembered 'left' When the server of an open ticket was skipped because an earlier answer said the member had left (they may have come back, and without the Server Members intent no join event arrives), a follow-up that finds exactly one other server now gets the picker instead of a new ticket there. --- src/bot/events/dmTickets.js | 6 ++++-- src/modules/tickets.js | 9 ++++++--- test/ticketGuildLookup.test.js | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/bot/events/dmTickets.js b/src/bot/events/dmTickets.js index 30b4752d..0585ff34 100644 --- a/src/bot/events/dmTickets.js +++ b/src/bot/events/dmTickets.js @@ -69,7 +69,7 @@ async function handleDM(message) { if (!payload.content && payload.attachments.length === 0) return; // Only the servers on the platform this DM arrived on: a user id means nothing on the other one. - const { guilds, openGuilds, openComplete, incomplete } = await lookupTicketGuilds( + const { guilds, openGuilds, openComplete, openRemembered, incomplete } = await lookupTicketGuilds( message.author, message.client ); @@ -96,7 +96,9 @@ async function handleDM(message) { .catch(() => {}); return; } - if (openGuilds.length === 0 && guilds.length === 1) { + // Not when an open ticket server was skipped from memory ("left"): the member may have come back + // to it, and a new ticket in the one other server could be the wrong one. They pick instead. + if (openGuilds.length === 0 && guilds.length === 1 && !openRemembered) { await ingestUserDM(guilds[0], message.author, payload); await message.react('✅').catch(() => {}); return; diff --git a/src/modules/tickets.js b/src/modules/tickets.js index bd288eca..6517bf49 100644 --- a/src/modules/tickets.js +++ b/src/modules/tickets.js @@ -316,6 +316,7 @@ async function findTicketGuilds(user, client) { // 1. Open tickets, from the database: complete, and kept across restarts. let openComplete = true; + let openRemembered = false; let openProbes = 0; const handled = new Set(); for (const key of await openTicketGuildKeys(user.id)) { @@ -327,6 +328,7 @@ async function findTicketGuilds(user, client) { if (isMember === null) { if ((notMember.get(`${user.id}|${key}`) ?? 0) > Date.now()) { isMember = false; // left, or refused: remembered + openRemembered = true; // not a fresh answer: the member may have come back } else if ( openProbes >= OPEN_PROBES_PER_DM || !takeWindow(openWindows, user.id, OPEN_PROBES_PER_USER_PER_MINUTE) @@ -384,17 +386,18 @@ async function findTicketGuilds(user, client) { if (found.length) { foundBefore.set(user.id, { guilds: new Set(found.map(guildKey)), at: Date.now() }); } - return { guilds: found, openGuilds, openComplete, incomplete }; + return { guilds: found, openGuilds, openComplete, openRemembered, incomplete }; } /** * Where can this sender open or continue a ticket? `guilds`: servers with tickets on that the user * is a member of; `openGuilds`: those where the user has an open ticket, and `openComplete`: that - * list is certain; `incomplete`: some servers could not be checked yet (the caps), so `guilds` may + * list is certain; `openRemembered`: an open ticket server was skipped on the strength of an + * earlier answer ("left"), not a fresh one; `incomplete`: some servers could not be checked yet (the caps), so `guilds` may * be missing some. Pass the client the DM arrived on to look only at that platform's servers. * A complete answer for one user is reused for a few seconds, so a burst of DMs does the work * once; an incomplete one is not reused, so a retry makes progress. - * @returns {Promise<{ guilds: import('discord.js').Guild[], openGuilds: import('discord.js').Guild[], openComplete: boolean, incomplete: boolean }>} + * @returns {Promise<{ guilds: import('discord.js').Guild[], openGuilds: import('discord.js').Guild[], openComplete: boolean, openRemembered: boolean, incomplete: boolean }>} */ export function lookupTicketGuilds(user, client) { const key = `${client ? platformOfClient(client) : 'all'}|${user.id}`; diff --git a/test/ticketGuildLookup.test.js b/test/ticketGuildLookup.test.js index 5b50de46..a9008b35 100644 --- a/test/ticketGuildLookup.test.js +++ b/test/ticketGuildLookup.test.js @@ -541,3 +541,39 @@ test('"not a member" is remembered shortly, a refused lookup for long, and a joi [plain.id] ); }); + +test('an open ticket server remembered as "left" does not decide an automatic new ticket elsewhere', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 25_000_000 }); + const user = { id: '780000000000000001' }; + const fetches = []; + const x = await ticketGuild(fetches); // the member has an open ticket here, and is not a member + await createTicket(x.id, user.id); + const y = await ticketGuild(fetches, { cached: [user.id] }); // the only other server + const guilds = [x, y]; + const replies = []; + const events = new EventEmitter(); + register(events); + + // First DM: a fresh answer says not a member of X, so the one server found is used. + events.emit('messageCreate', dmFrom(user, guilds, replies)); + await settle(); + assert.ok(await getOpenTicket(y.id, user.id), 'a fresh answer: opened in the one server found'); + + // Second member, same situation, but the answer for X comes from memory (they may have come back). + const user2 = { id: '780000000000000002' }; + const x2 = await ticketGuild(fetches); + await createTicket(x2.id, user2.id); + const y2 = await ticketGuild(fetches, { cached: [user2.id] }); + const first = await lookupTicketGuilds(user2, clientOf([x2, y2])); + assert.equal(first.openRemembered, false); + t.mock.timers.tick(LOOKUP_REUSE_MS + 1); + const second = await lookupTicketGuilds(user2, clientOf([x2, y2])); + assert.equal(second.openRemembered, true); + const before = replies.length; + events.emit('messageCreate', dmFrom(user2, [x2, y2], replies)); + await settle(); + assert.equal(await getOpenTicket(y2.id, user2.id), undefined, 'nothing opened in the other server'); + assert.equal(replies.length, before + 1); + assert.ok(replies.at(-1).components, 'the member gets the picker'); +}); From 27877b442f9ae37651113340c946aeaf8ff592d1 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 03:01:07 +0100 Subject: [PATCH 17/21] fix(live-alerts): batch servers that save rejected values apart, ask the canary before the search - A server is flagged for two hours after one of its values is found rejected; its values are batched on their own, never with other servers' values, so a server that saves fresh rejected values every poll cannot keep others unresolved. Servers with no rejected value still share batches, which keeps the cost at one request per hundred values. - When no batch answers, the canary is asked before the search budget is spent, and a refused canary fails the poll at once. - The comment describes the order the batches are cut and searched in. --- src/modules/kickAlerts.js | 11 +++- src/modules/lib/batchIsolation.js | 93 ++++++++++++++++++++++++++----- src/modules/twitchAlerts.js | 11 +++- test/liveAlertsIsolation.test.js | 85 ++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 18 deletions(-) diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index a7e5506a..f5a3a6a8 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -114,7 +114,7 @@ export const rejectedSlugs = createQuarantine(); * `report({ rejected, unresolved, inconclusive, skipped })` is called once with the slugs rejected now and every slug * this poll knows nothing about. */ -export async function fetchLiveChannels(slugs, { report } = {}) { +export async function fetchLiveChannels(slugs, { report, groupOf } = {}) { const out = new Map(); const { results, rejected, unresolved, inconclusive, skipped } = await fetchIsolated( slugs, @@ -122,6 +122,7 @@ export async function fetchLiveChannels(slugs, { report } = {}) { getChannels, { quarantine: rejectedSlugs, + groupOf, } ); report?.({ rejected, unresolved, inconclusive, skipped }); @@ -195,11 +196,19 @@ export async function tick() { if (!jobs.length) return; const slugs = [...new Set(jobs.map((j) => j.alert.slug))]; + // The server that saved each slug (the lowest id, if several did): rejected values are batched + // by it, see lib/batchIsolation.js. + const savedBy = new Map(); + for (const j of jobs) { + if (!savedBy.has(j.alert.slug) || j.guildId < savedBy.get(j.alert.slug)) + savedBy.set(j.alert.slug, j.guildId); + } rejectedSlugs.prune(new Set(slugs)); let live; let unresolved = new Set(); try { live = await fetchLiveChannels(slugs, { + groupOf: (slug) => savedBy.get(slug) ?? null, report: ({ rejected, unresolved: left, inconclusive, skipped }) => { unresolved = left; if (skipped > 0) { diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js index 16703fe9..7b95ca4d 100644 --- a/src/modules/lib/batchIsolation.js +++ b/src/modules/lib/batchIsolation.js @@ -5,14 +5,23 @@ // poll for every other server. fetchIsolated() works in two phases: // 1. one request per batch. A batch that answers is done, and counts as evidence that the request // itself is fine. This always fits the budget, so a clean batch is never left out because -// another one needed a long search. +// another one needed a long search. Batches are cut in this order: the values of groups (servers) +// not known to save rejected values, in one pool; then the suspects (see below) of those +// groups; then, for each flagged group, its own values on their own. If no batch answered, one +// canary request (below) is sent before phase 2, and a refused canary fails the poll at once. // 2. only the batches that failed are halved until the rejected items stand alone. This has its // own budget (EXTRA_REQUESTS) and goes depth first: one failed batch is searched down to single // values before the next is started, so every poll isolates (and so quarantines) values, however -// many there are. When the budget runs out, the items of the chunks not reached are left out of -// this poll (never failed) and remembered as suspects: the next poll batches them apart from -// the clean values, so the clean ones are always fetched, and searches them first. A poll never -// fails because the budget ran out. +// many there are. Failed batches are searched in the order they were cut, so the pool's failures +// come before the suspects' and the flagged groups'. When the budget runs out, the items of the +// chunks not reached are left out of this poll (never failed) and remembered as suspects: the +// next poll batches them after the clean values, so the clean ones are always fetched. A poll +// never fails because the budget ran out. +// A group (the server that saved a value, `groupOf`) is flagged when one of its values was found +// rejected, for FLAG_TTL_MS after the last time. Its values are never batched with other groups' +// values again while flagged, so a server that keeps saving fresh rejected values (150 a poll) +// cannot keep other servers' values unresolved. Not every group is batched apart all the time +// because that would cost one request per server per poll, however small. // The values found are quarantined with a growing backoff after the poll. // // Item fault or request fault? Only the statuses that can mean "this value" count (400, 404, 422); @@ -26,6 +35,9 @@ /** Requests per poll, beyond one per batch, for finding the rejected items. */ export const EXTRA_REQUESTS = 120; +/** How long a group stays flagged after one of its values was last found rejected. */ +export const FLAG_TTL_MS = 2 * 60 * 60_000; + /** Statuses that can mean "this value was refused", as against a problem with the request. */ const VALUE_STATUSES = new Set([400, 404, 422]); @@ -48,6 +60,8 @@ export function createQuarantine({ let turn = 0; /** @type {Set} */ let suspects = new Set(); + /** @type {Map} group -> when its flag runs out */ + const flagged = new Map(); return { /** True while the item is waiting out its backoff. */ has(key) { @@ -83,6 +97,19 @@ export function createQuarantine({ setGood(item) { good = item; }, + /** Flag a group (a server that saved a rejected value) for FLAG_TTL_MS from now. */ + flag(group) { + flagged.set(group, now() + FLAG_TTL_MS); + }, + isFlagged(group) { + const until = flagged.get(group); + if (until === undefined) return false; + if (until <= now()) { + flagged.delete(group); + return false; + } + return true; + }, /** Values left unchecked inside a failed chunk when the last poll ran out of budget. */ get suspects() { return suspects; @@ -106,7 +133,7 @@ export function createQuarantine({ * @param {string[]} items * @param {number} size * @param {(batch: string[]) => Promise} fetchBatch throws an error with `.status` on a non-OK answer - * @param {{ quarantine: ReturnType, extraRequests?: number }} o + * @param {{ quarantine: ReturnType, extraRequests?: number, groupOf?: (item: string) => string | null }} o * @returns {Promise<{ * results: T[], * rejected: Array<{ item: string, err: Error, waitMs: number, strikes: number }>, @@ -116,7 +143,12 @@ export function createQuarantine({ * }>} `skipped`: how many items were left out only because the budget ran out. `unresolved`: the items this poll knows nothing about (quarantined before, rejected now, or * not reached when the budget ran out); their stored state must be left alone. */ -export async function fetchIsolated(items, size, fetchBatch, { quarantine, extraRequests = EXTRA_REQUESTS }) { +export async function fetchIsolated( + items, + size, + fetchBatch, + { quarantine, extraRequests = EXTRA_REQUESTS, groupOf = () => null } +) { const results = []; const unresolved = new Set(); const found = []; // rejected on their own: { item, err } @@ -146,25 +178,52 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extra unresolved.add(item); return false; }); - // Values left unchecked in a failed chunk by an earlier poll go in batches of their own, after - // the others, so clean values never wait behind them. - const suspect = live.filter((item) => quarantine.suspects.has(item)); - const clean = live.filter((item) => !quarantine.suspects.has(item)); - const start = quarantine.nextStart(clean.length, size); + // Cut the batches: the pool of unflagged groups first, then its suspects (values left unchecked + // in a failed chunk by an earlier poll), then each flagged group on its own. + const flaggedItems = new Map(); // group -> items + const pool = []; + const poolSuspects = []; + for (const item of live) { + const group = groupOf(item); + if (group !== null && quarantine.isFlagged(group)) { + if (!flaggedItems.has(group)) flaggedItems.set(group, []); + flaggedItems.get(group).push(item); + } else if (quarantine.suspects.has(item)) poolSuspects.push(item); + else pool.push(item); + } + const start = quarantine.nextStart(pool.length, size); const batches = []; - for (const group of [[...clean.slice(start), ...clean.slice(0, start)], suspect]) { - for (let i = 0; i < group.length; i += size) batches.push(group.slice(i, i + size)); + for (const list of [ + [...pool.slice(start), ...pool.slice(0, start)], + poolSuspects, + ...flaggedItems.values(), + ]) { + for (let i = 0; i < list.length; i += size) batches.push(list.slice(i, i + size)); } // Phase 1: one request per batch. const failed = []; + let firstFailure = null; for (const batch of batches) { const answer = await ask(batch); if (answer === true) continue; + firstFailure ??= answer; if (batch.length === 1) found.push({ item: batch[0], err: answer }); else failed.push(batch); } + // Nothing answered: before spending the search budget, ask the canary. Refused too means the + // request is at fault, and the poll fails now instead of after 120 more refused requests. + let canaryAnswered = false; + if (succeeded === 0 && firstFailure !== null && canary) { + const answer = await ask([canary], false).catch((err) => err); + if (answer !== true) { + quarantine.setGood(null); + throw firstFailure; + } + canaryAnswered = true; + } + // Phase 2: search the failed batches depth first until the budget is used up. Both halves of a // chunk are asked; the failed ones go on a stack with the first half on top. let spare = extraRequests; @@ -197,7 +256,9 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extra let inconclusive = false; let quarantineFound = found.length > 0; if (quarantineFound) { - if (canary && !found.some((f) => f.item === canary)) { + if (canaryAnswered) { + // already asked, and the platform answers it: the values are at fault + } else if (canary && !found.some((f) => f.item === canary)) { // Is the platform refusing everything? One request with a value that was fine before. A // failure of this request of any kind counts as "refused": it must not throw away what the // poll already fetched. @@ -221,6 +282,8 @@ export async function fetchIsolated(items, size, fetchBatch, { quarantine, extra for (const { item, err } of found) { unresolved.add(item); if (!quarantineFound) continue; + const group = groupOf(item); + if (group !== null) quarantine.flag(group); const waitMs = quarantine.add(item); rejected.push({ item, err, waitMs, strikes: quarantine.strikes(item) }); } diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 4bc5ca4a..7a1056a2 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -108,13 +108,13 @@ export const rejectedLogins = createQuarantine(); * isolated (see fetchIsolated) and left out. `report({ rejected, unresolved, inconclusive, skipped })` is called once with * the logins rejected now and every login this poll knows nothing about. */ -export async function fetchLiveStreams(logins, { report } = {}) { +export async function fetchLiveStreams(logins, { report, groupOf } = {}) { const out = new Map(); const { results, rejected, unresolved, inconclusive, skipped } = await fetchIsolated( logins, 100, (batch) => helix('/streams', { user_login: batch }), - { quarantine: rejectedLogins } + { quarantine: rejectedLogins, groupOf } ); report?.({ rejected, unresolved, inconclusive, skipped }); for (const s of results) out.set(s.user_login.toLowerCase(), s); @@ -199,12 +199,19 @@ export async function tick() { if (!jobs.length) return; const logins = [...new Set(jobs.map((j) => j.alert.login))]; + // The server that saved each login (the lowest id, if several did): rejected values are batched + // by it, see lib/batchIsolation.js. + const savedBy = new Map(); + for (const j of jobs) + if (!savedBy.has(j.alert.login) || j.guildId < savedBy.get(j.alert.login)) + savedBy.set(j.alert.login, j.guildId); rejectedLogins.prune(new Set(logins)); let streams; let users; let unresolved = new Set(); try { streams = await fetchLiveStreams(logins, { + groupOf: (login) => savedBy.get(login) ?? null, report: ({ rejected, unresolved: left, inconclusive, skipped }) => { unresolved = left; if (skipped > 0) { diff --git a/test/liveAlertsIsolation.test.js b/test/liveAlertsIsolation.test.js index f4ce2cd6..7fb44399 100644 --- a/test/liveAlertsIsolation.test.js +++ b/test/liveAlertsIsolation.test.js @@ -799,3 +799,88 @@ test('kick tick: 250 poisoned slugs in five guilds converge and a guild behind t runtime.client = null; } }); + +// --- round 5: fresh poison every poll from a few servers, the canary before the search ----------- + +for (const [label, size, perGuild] of [ + ['Twitch: 150 new rejected logins per poll from 3 servers', 100, 50], + ['Kick: 100 new rejected slugs per poll from 3 servers', 50, 34], +]) { + test(`fetchIsolated: ${label} do not keep other servers' values unresolved`, async () => { + const quarantine = createQuarantine(); + const victims = Array.from({ length: 20 }, (_, i) => `victim${i}`); + const owner = new Map(); + victims.forEach((v, i) => owner.set(v, `v${i % 2}`)); + const ask = async (batch) => { + if (batch.some((i) => i.startsWith('_'))) throw rejected(400); + return batch; + }; + const fetchedEveryPoll = []; + for (let poll = 1; poll <= 8; poll += 1) { + // The same three servers save new rejected values every poll. + const fresh = []; + for (const g of ['a', 'b', 'c']) { + for (let i = 0; i < perGuild; i += 1) { + const name = `_${g}${poll}x${i}`; + owner.set(name, `bad-${g}`); + fresh.push(name); + } + } + const items = [...fresh.slice(0, perGuild), ...victims, ...fresh.slice(perGuild)]; + const out = await fetchIsolated(items, size, ask, { quarantine, groupOf: (i) => owner.get(i) ?? null }); + fetchedEveryPoll.push(victims.every((v) => out.results.includes(v))); + } + assert.ok( + fetchedEveryPoll.slice(1).every(Boolean), + `victims fetched in every poll after the first: ${fetchedEveryPoll.join(',')}` + ); + }); +} + +test('fetchIsolated: a server is flagged when one of its values is found rejected, and only then', async () => { + const quarantine = createQuarantine(); + const ask = async (batch) => { + if (batch.includes('_bad')) throw rejected(400); + return batch; + }; + const group = (i) => (i === '_bad' ? 'guild-a' : 'guild-b'); + await fetchIsolated(['_bad', 'ok1', 'ok2'], 10, ask, { quarantine, groupOf: group }); + assert.equal(quarantine.isFlagged('guild-a'), true); + assert.equal(quarantine.isFlagged('guild-b'), false); +}); + +test('fetchIsolated: when no batch answers and the canary is refused too, the poll fails before the search', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const items = Array.from({ length: 300 }, (_, i) => `_x${i}`); + let calls = 0; + await assert.rejects( + fetchIsolated( + items, + 100, + async () => { + calls += 1; + throw rejected(400); + }, + { quarantine } + ), + /rejected 400/ + ); + assert.equal(calls, 3 + 1, 'three batches and the canary, no search'); + assert.equal(quarantine.size, 0); +}); + +test('fetchIsolated: when no batch answers but the canary does, the search runs and the values are quarantined', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const out = await fetchIsolated( + ['_x1', '_x2'], + 10, + async (batch) => { + if (batch.every((i) => i === 'known-good')) return batch; + throw rejected(400); + }, + { quarantine } + ); + assert.deepEqual(out.rejected.map((r) => r.item).sort(), ['_x1', '_x2']); +}); From 12571cc673081aa824a4599849290bf652a33352 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 03:01:07 +0100 Subject: [PATCH 18/21] fix(tickets): offer the remembered server in the picker and let the pick ask again When the picker shows because an open ticket server was skipped from memory, that server is one of the options. Picking a server where the member has an open ticket asks Discord once even if it is remembered as left, and a fresh member answer clears the memory. --- src/bot/events/dmTickets.js | 9 ++++----- src/modules/tickets.js | 20 +++++++++++++++----- test/ticketGuildLookup.test.js | 22 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/bot/events/dmTickets.js b/src/bot/events/dmTickets.js index 0585ff34..3a79c501 100644 --- a/src/bot/events/dmTickets.js +++ b/src/bot/events/dmTickets.js @@ -69,10 +69,8 @@ async function handleDM(message) { if (!payload.content && payload.attachments.length === 0) return; // Only the servers on the platform this DM arrived on: a user id means nothing on the other one. - const { guilds, openGuilds, openComplete, openRemembered, incomplete } = await lookupTicketGuilds( - message.author, - message.client - ); + const { guilds, openGuilds, openComplete, openRemembered, rememberedGuilds, incomplete } = + await lookupTicketGuilds(message.author, message.client); // The open tickets come from the database, so they are certain. A follow-up goes to the one // server where the member has an open ticket, however much of the rest could be checked. @@ -110,7 +108,8 @@ async function handleDM(message) { .setCustomId(SELECT_ID) .setPlaceholder('Which server is this about?') .addOptions( - (openGuilds.length ? openGuilds : guilds) + // A server skipped from memory ("left") is offered too: the member may have come back to it. + [...new Set([...(openGuilds.length ? openGuilds : guilds), ...rememberedGuilds])] .slice(0, 25) .map((g) => ({ label: g.name.slice(0, 100), value: guildKey(g) })) ); diff --git a/src/modules/tickets.js b/src/modules/tickets.js index 6517bf49..85a79062 100644 --- a/src/modules/tickets.js +++ b/src/modules/tickets.js @@ -9,6 +9,7 @@ import { isModuleEnabled, getGuildModule } from '../db/modules.js'; import { notifyTicketActivity } from '../lib/push.js'; import { sweepMap } from '../lib/sweepMap.js'; import { + getOpenTicket, openTicketGuildKeys, openOrCreateTicket, getTicket, @@ -289,7 +290,10 @@ async function probeMember(guild, userId, { ttl = NOT_MEMBER_TTL_MS } = {}) { checkedAt.set(`${userId}|${guildKey(guild)}`, Date.now()); try { // A platform may answer "no such member" with null instead of an error. - if (await guild.members.fetch(userId)) return true; + if (await guild.members.fetch(userId)) { + notMember.delete(`${userId}|${guildKey(guild)}`); // they are a member after all + return true; + } remember(ttl); return false; } catch (err) { @@ -317,6 +321,7 @@ async function findTicketGuilds(user, client) { // 1. Open tickets, from the database: complete, and kept across restarts. let openComplete = true; let openRemembered = false; + const rememberedGuilds = []; let openProbes = 0; const handled = new Set(); for (const key of await openTicketGuildKeys(user.id)) { @@ -329,6 +334,7 @@ async function findTicketGuilds(user, client) { if ((notMember.get(`${user.id}|${key}`) ?? 0) > Date.now()) { isMember = false; // left, or refused: remembered openRemembered = true; // not a fresh answer: the member may have come back + rememberedGuilds.push(guild); } else if ( openProbes >= OPEN_PROBES_PER_DM || !takeWindow(openWindows, user.id, OPEN_PROBES_PER_USER_PER_MINUTE) @@ -386,18 +392,18 @@ async function findTicketGuilds(user, client) { if (found.length) { foundBefore.set(user.id, { guilds: new Set(found.map(guildKey)), at: Date.now() }); } - return { guilds: found, openGuilds, openComplete, openRemembered, incomplete }; + return { guilds: found, openGuilds, openComplete, openRemembered, rememberedGuilds, incomplete }; } /** * Where can this sender open or continue a ticket? `guilds`: servers with tickets on that the user * is a member of; `openGuilds`: those where the user has an open ticket, and `openComplete`: that * list is certain; `openRemembered`: an open ticket server was skipped on the strength of an - * earlier answer ("left"), not a fresh one; `incomplete`: some servers could not be checked yet (the caps), so `guilds` may + * earlier answer ("left"), not a fresh one, and `rememberedGuilds` are those servers; `incomplete`: some servers could not be checked yet (the caps), so `guilds` may * be missing some. Pass the client the DM arrived on to look only at that platform's servers. * A complete answer for one user is reused for a few seconds, so a burst of DMs does the work * once; an incomplete one is not reused, so a retry makes progress. - * @returns {Promise<{ guilds: import('discord.js').Guild[], openGuilds: import('discord.js').Guild[], openComplete: boolean, openRemembered: boolean, incomplete: boolean }>} + * @returns {Promise<{ guilds: import('discord.js').Guild[], openGuilds: import('discord.js').Guild[], openComplete: boolean, openRemembered: boolean, rememberedGuilds: import('discord.js').Guild[], incomplete: boolean }>} */ export function lookupTicketGuilds(user, client) { const key = `${client ? platformOfClient(client) : 'all'}|${user.id}`; @@ -434,7 +440,11 @@ export async function canOpenTicketIn(user, guild) { // A server the member left (or that refuses lookups) stays refused for a while, and a pick costs // at most one fetch from the user's own small allowance: clicking the menu again and again is // bounded. - if ((notMember.get(`${user.id}|${guildKey(guild)}`) ?? 0) > Date.now()) return false; + // The one exception is a server where the member has an open ticket: it was offered in the menu + // because they may have come back, and the pick asks Discord once to find out. + if ((notMember.get(`${user.id}|${guildKey(guild)}`) ?? 0) > Date.now()) { + if (!(await getOpenTicket(guildKey(guild), user.id))) return false; + } if (!takeWindow(openWindows, user.id, OPEN_PROBES_PER_USER_PER_MINUTE)) return false; return (await probeMember(guild, user.id)) === true; } diff --git a/test/ticketGuildLookup.test.js b/test/ticketGuildLookup.test.js index a9008b35..7f7cfcd1 100644 --- a/test/ticketGuildLookup.test.js +++ b/test/ticketGuildLookup.test.js @@ -576,4 +576,26 @@ test('an open ticket server remembered as "left" does not decide an automatic ne assert.equal(await getOpenTicket(y2.id, user2.id), undefined, 'nothing opened in the other server'); assert.equal(replies.length, before + 1); assert.ok(replies.at(-1).components, 'the member gets the picker'); + const options = replies.at(-1).components[0].components[0].options.map((o) => o.data.value); + assert.deepEqual(options.sort(), [x2.id, y2.id].sort(), 'the remembered server is an option too'); + + // Picking it: the member came back, so asking Discord once finds them there. + runtime.client = clientOf([x2, y2]); + t.after(() => { + runtime.client = null; + }); + x2.members.fetch = async (id) => ({ id }); + const sent = []; + await routeComponent({ + customId: 'ticket-guild', + values: [x2.id], + user: { ...user2, tag: 'tester#0002', send: async () => {} }, + client: runtime.client, + isRepliable: () => true, + update: async (x) => sent.push(x), + reply: async (x) => sent.push(x), + }); + assert.match(sent.at(-1).content, /Opened ticket/); + const ticket = await getOpenTicket(x2.id, user2.id); + assert.equal((await ticketMessages(ticket.id)).length, 1, 'the message went to the open ticket'); }); From 47263a1a464f963422a959e656ac1315f01c91c8 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 22:58:09 +0100 Subject: [PATCH 19/21] test(backups): pin db.roleRisks in the health report for app clients The iOS app detects restore-confirmation support (confirm + acknowledgeSql) by db.roleRisks being an array in the health report. Pin that contract with a route test and note it in the mobile app docs. --- docs/mobile-app.md | 3 +++ test/v2HealthPlatforms.test.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/docs/mobile-app.md b/docs/mobile-app.md index 885c48fc..35b6ab7c 100644 --- a/docs/mobile-app.md +++ b/docs/mobile-app.md @@ -617,6 +617,9 @@ its account is an operator. is stored as a snapshot only. The import is sent as raw `application/octet-stream` (a JSON body is cut at 256 kB) of up to 128 MB, read from the file as it goes. + A client detects that the server asks for the restore confirmation (`confirm` and + `acknowledgeSql`) by `db.roleRisks` being an array in the health report from + `GET /api/v2/health` (5.3 and later); an older server does not send it. - **Bot Personalizer**: the bot's name and pictures, its presence, and the announcement everyone sees. - **Stats cache**: the game-stats lookups every server shares (read only). diff --git a/test/v2HealthPlatforms.test.js b/test/v2HealthPlatforms.test.js index e45dbfcc..b28f7753 100644 --- a/test/v2HealthPlatforms.test.js +++ b/test/v2HealthPlatforms.test.js @@ -78,3 +78,31 @@ test('/api/v2/health names the module an error came from', async () => { runtime.errors.splice(0, runtime.errors.length, ...saved); } }); + +// The iOS app tells that a server asks for the restore confirmation (`confirm` + `acknowledgeSql`) +// by `db.roleRisks` being an array in this report, next to the backups list. Keep both in place. +test('/api/v2/health carries db.roleRisks as an array next to the backups', async () => { + const { config } = await import('../src/config.js'); + const { checkDatabaseRole, lastRolePrivileges, roleRisks } = await import('../src/db/dbRole.js'); + await checkDatabaseRole({ warn() {} }); + const { accountId } = await signInIdentity({ + platform: 'discord', + platformUserId: '111111111111111111', + username: 'u', + avatar: null, + guildIds: [], + }); + const { token } = await createApiToken(accountId, { deviceName: 'iPhone', ttlMs: 60_000 }); + const res = await browser().get('/api/v2/health', { headers: { authorization: `Bearer ${token}` } }); + assert.equal(res.status, 200); + const body = await res.json(); + assert.ok(Array.isArray(body.backups), 'the report lists backups'); + assert.ok(Array.isArray(body.db.roleRisks), 'db.roleRisks is an array, never missing or null'); + assert.deepEqual(body.db.roleRisks, roleRisks(lastRolePrivileges())); + if (!config.databaseUrl) { + assert.deepEqual(body.db.roleRisks, [], 'SQLite has no database role, so no risks'); + } else if (lastRolePrivileges()?.superuser) { + // The test database role is the Postgres superuser: the risks are listed, not hidden. + assert.ok(body.db.roleRisks.includes('SUPERUSER')); + } +}); From c8c86e03b4221913a48961ae15a9dbf3d57a43ef Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 23:44:27 +0100 Subject: [PATCH 20/21] fix(health): show the database role warning first in its section and as a warning in the restore dialog The role warning moves to the top of Database & backups and says in one line when the role is a superuser, with the full list of rights folded behind Details. In the restore dialog the SQL warning for an uploaded file is a warning callout with the same short wording, and the link out of the dialog is gone (the page callout has it). --- web-v2/src/__tests__/Health.test.jsx | 30 +++++++++-- web-v2/src/pages/Health.jsx | 76 ++++++++++++++++++---------- web-v2/src/styles/health.css | 27 ++++++++++ 3 files changed, 104 insertions(+), 29 deletions(-) diff --git a/web-v2/src/__tests__/Health.test.jsx b/web-v2/src/__tests__/Health.test.jsx index e5d59c1d..f591282d 100644 --- a/web-v2/src/__tests__/Health.test.jsx +++ b/web-v2/src/__tests__/Health.test.jsx @@ -117,8 +117,23 @@ describe('Health: restoring an uploaded Postgres dump', () => { const { container } = dialogBody(0); expect(container.textContent).toMatch(/This file was uploaded/); expect(container.textContent).toMatch(/runs that SQL with the rights of the database role/); - expect(container.textContent).toMatch(/SUPERUSER/); - expect(container.querySelector('a')?.getAttribute('href')).toMatch(/least-privilege-database-role/); + // short and plain; the full list of rights is folded away, and the way out is not offered in the dialog + expect(container.textContent).toMatch(/This role is a superuser\./); + expect(container.querySelector('details')?.textContent).toMatch(/SUPERUSER/); + expect(container.querySelector('a')).toBeNull(); + }); + + it('shows the uploaded-file warning as a warning callout, not as plain text', async () => { + getHealth.mockResolvedValue(pgHealth({ roleRisks: ['CREATEDB', 'CREATEROLE'] })); + render(); + await screen.findByText('sylo-imported-20261116.dump'); + fireEvent.click(restoreButtons()[0]); + await waitFor(() => expect(confirmDialog).toHaveBeenCalledTimes(1)); + const { container } = dialogBody(0); + const callout = container.querySelector('.v2-callout.is-warning'); + expect(callout?.textContent).toMatch(/This file was uploaded/); + expect(callout?.textContent).toMatch(/This role has more rights than it needs\./); + expect(callout?.textContent).toMatch(/Rights: CREATEDB, CREATEROLE/); }); it.each([ @@ -151,7 +166,16 @@ describe('Health: restoring an uploaded Postgres dump', () => { it('shows a note on the page when the role is stronger than needed, and not otherwise', async () => { getHealth.mockResolvedValue(pgHealth({ roleRisks: ['SUPERUSER', 'CREATEDB'] })); const first = render(); - expect(await screen.findByText(/more rights than Sylo needs/i)).toBeTruthy(); + const title = await screen.findByText(/more rights than Sylo needs/i); + // a warning callout, the first thing in its section (before the database summary and the buttons) + const callout = title.closest('.v2-callout'); + expect(callout?.classList.contains('is-warning')).toBe(true); + const section = callout.parentElement; + expect(section.querySelector('h2').textContent).toMatch(/Database/); + expect(section.children[1]).toBe(callout); + expect(callout.textContent).toMatch(/The role Sylo connects as is a superuser\./); + expect(callout.querySelector('details')?.textContent).toMatch(/Rights: SUPERUSER, CREATEDB/); + expect(callout.querySelector('a')?.getAttribute('href')).toMatch(/least-privilege-database-role/); first.unmount(); getHealth.mockResolvedValue(pgHealth()); diff --git a/web-v2/src/pages/Health.jsx b/web-v2/src/pages/Health.jsx index 608f3fcf..fa9b2745 100644 --- a/web-v2/src/pages/Health.jsx +++ b/web-v2/src/pages/Health.jsx @@ -83,18 +83,34 @@ export const isImportedBackup = (name) => /^sylo-imported-/.test(name); * Said before an uploaded Postgres dump is restored: the SQL in the file is run by the database * with the rights of the role Sylo connects as. */ -function UploadedSqlWarning({ risks = [], docsUrl }) { +function UploadedSqlWarning({ risks = [] }) { return ( -

- This file was uploaded. A Postgres dump holds SQL, and restoring it runs that SQL with - the rights of the database role Sylo connects as - {risks.length ? <> (this role has {risks.join(', ')}) : null}. Restore only a file you made or trust.{' '} - {docsUrl ? ( - - Use a role with few rights - + +

+ A Postgres dump holds SQL. Restoring it runs that SQL with the rights of the database role Sylo + connects as. Restore only a file you made or trust. +

+ {risks.length ? ( + <> +

{roleSentence(risks)}

+ + ) : null} -

+ + ); +} + +/** The short version of what is wrong with the role: a superuser says it all, the rest is listed. */ +const roleSentence = (risks) => + risks.includes('SUPERUSER') ? 'This role is a superuser.' : 'This role has more rights than it needs.'; + +/** The full list of the role's extra rights, folded away. */ +function RoleDetails({ risks }) { + return ( +
+ Details +

Rights: {risks.join(', ')}

+
); } @@ -201,9 +217,7 @@ export default function Health() { All current data is replaced with {name} and the bot restarts. A prerestore snapshot is taken first.

- {d.db.postgres && isImportedBackup(name) ? ( - - ) : null} + {d.db.postgres && isImportedBackup(name) ? : null} ), confirmLabel: 'Restore and restart', @@ -437,6 +451,29 @@ export default function Health() {

Database & backups

+ {/* First in the section: it is about the database itself, and a restore below runs with this role. */} + {d.db.postgres && d.db.roleRisks?.length ? ( + +

+ {d.db.roleRisks.includes('SUPERUSER') + ? 'The role Sylo connects as is a superuser.' + : 'The role Sylo connects as has rights Sylo does not need.'}{' '} + A restored dump runs its SQL with those rights. +

+ + {d.db.roleDocsUrl ? ( +

+ + How to set up a role with few rights + +

+ ) : null} +
+ ) : null}

Database {d.db.size} {d.db.wal !== null ? ` (WAL ${d.db.wal})` : ''}.{' '} @@ -476,19 +513,6 @@ export default function Health() { {fileName ? null : Choose a file first.}

- {d.db.postgres && d.db.roleRisks?.length ? ( - -

- The role Sylo connects as has {d.db.roleRisks.join(', ')}. A restored dump runs its SQL with - those rights.{' '} - {d.db.roleDocsUrl ? ( - - How to set up a role with few rights - - ) : null} -

-
- ) : null}

Importing stores the file as a snapshot; you then Restore it. Restoring replaces the live database and restarts the bot (a prerestore snapshot is taken first). diff --git a/web-v2/src/styles/health.css b/web-v2/src/styles/health.css index 4b7aabb0..13ada8c0 100644 --- a/web-v2/src/styles/health.css +++ b/web-v2/src/styles/health.css @@ -75,3 +75,30 @@ .v2-module-health-notice:empty { display: none; } + +/* ---- Database role: the warning at the top of "Database & backups" and in the restore dialog ---------- */ +.v2-role-warning { + margin: 0 0 16px; +} +.v2-role-warning .v2-callout-title { + color: var(--text); +} +.v2-role-details { + margin-top: 6px; + color: var(--text-muted); +} +.v2-role-details summary { + display: inline-block; + cursor: pointer; + font-weight: 600; + text-decoration: underline; + text-underline-offset: 3px; +} +.v2-role-details p { + margin: 4px 0 0; + overflow-wrap: anywhere; +} +/* The warning in the dialog sits between the question and the type-to-confirm field. */ +.v2-restore-warning { + margin: 12px 0 0; +} From 2a90488a532b5c488e1254607e24459c95bb6023 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Tue, 17 Nov 2026 00:20:59 +0100 Subject: [PATCH 21/21] fix(tickets): accept a DM menu pick only from the DM's own platform The pick was resolved on any platform and checked with the picked server's client. The menu only offers servers of the DM's platform and select values are validated upstream, so this is defence in depth: a value from another platform is refused like any pick that cannot be confirmed. --- src/bot/events/dmTickets.js | 9 ++++-- test/ticketGuildLookup.test.js | 57 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/bot/events/dmTickets.js b/src/bot/events/dmTickets.js index 3a79c501..e6a277ed 100644 --- a/src/bot/events/dmTickets.js +++ b/src/bot/events/dmTickets.js @@ -2,7 +2,7 @@ // a DM questionnaire (bot/lib/dmQuestionnaire.js), which takes their DMs first. When the // user shares several ticket-enabled servers and has no open ticket, the bot // asks which server with a select menu. -import { guildKey, guildByKey, platformOfClient } from '../../platform/platformOf.js'; +import { guildKey, guildByKey, platformOfClient, platformOfGuild } from '../../platform/platformOf.js'; import { config } from '../../config.js'; import { Events, StringSelectMenuBuilder, ActionRowBuilder } from 'discord.js'; import { lookupTicketGuilds, canOpenTicketIn, forgetNotMember, ingestUserDM } from '../../modules/tickets.js'; @@ -131,8 +131,11 @@ async function handleSelect(interaction) { .catch(() => {}); return; } - // Still one the member may open a ticket in (a member there, tickets on): the menu could be old. - const guild = picked && (await canOpenTicketIn(interaction.user, picked)) ? picked : null; + // Only a server on the platform this menu was shown on: a user id means nothing on the other one (the menu + // only offers those, so this guards against a value that was not from it). Then still one the member may + // open a ticket in (a member there, tickets on): the menu could be old. + const samePlatform = picked && platformOfGuild(picked) === platformOfClient(interaction.client); + const guild = samePlatform && (await canOpenTicketIn(interaction.user, picked)) ? picked : null; if (!guild) { // The message stays stashed and the menu stays up, so the member can pick again. await interaction diff --git a/test/ticketGuildLookup.test.js b/test/ticketGuildLookup.test.js index 7f7cfcd1..e4ca7862 100644 --- a/test/ticketGuildLookup.test.js +++ b/test/ticketGuildLookup.test.js @@ -599,3 +599,60 @@ test('an open ticket server remembered as "left" does not decide an automatic ne const ticket = await getOpenTicket(x2.id, user2.id); assert.equal((await ticketMessages(ticket.id)).length, 1, 'the message went to the open ticket'); }); + +test('a menu pick of a server on the other platform is refused, and nothing is opened there', async (t) => { + _resetTicketLookup(); + t.mock.timers.enable({ apis: ['Date'], now: 30_000_000 }); + const user = { id: '760000000000000001' }; + const fetches = []; + const a = await ticketGuild(fetches, { cached: [user.id] }); + const b = await ticketGuild(fetches, { cached: [user.id] }); + // A Fluxer server where the same id is a member and tickets are on: the pick would pass every other check. + const fluxerClient = {}; + const raw = String(nextId++); + const foreign = { + id: raw, + name: 'Fluxer server', + client: fluxerClient, + memberCount: 5000, + channels: { cache: new Map(), fetch: async () => null }, + members: { + cache: new Map([[user.id, { id: user.id, partial: false }]]), + fetch: async () => ({ id: user.id }), + }, + }; + fluxerClient.guilds = { cache: new Map([[raw, foreign]]) }; + await setGuildModule(`fluxer:${raw}`, 'tickets', { enabled: true }); + runtime.client = clientOf([a, b]); + runtime.clients.fluxer = fluxerClient; + t.after(() => { + runtime.client = null; + runtime.clients.fluxer = null; + }); + const replies = []; + const client = new EventEmitter(); + register(client); + client.emit('messageCreate', dmFrom(user, [a, b], replies)); + await settle(); + assert.ok(replies[0].components, 'the picker'); + + const sent = []; + const pick = (value, from) => ({ + customId: 'ticket-guild', + values: [value], + user: { ...user, tag: 'tester#0003', send: async () => {} }, + client: from, + isRepliable: () => true, + update: async (x) => sent.push(['update', x]), + reply: async (x) => sent.push(['reply', x]), + }); + // The DM came in on Discord: the Fluxer server's key is not one it may pick. + await routeComponent(pick(`fluxer:${raw}`, runtime.client)); + assert.equal(sent.at(-1)[0], 'reply'); + assert.match(sent.at(-1)[1].content, /could not confirm/); + assert.equal(await getOpenTicket(`fluxer:${raw}`, user.id), undefined, 'nothing opened there'); + // The message was kept: a server on the DM's own platform still works. + await routeComponent(pick(a.id, runtime.client)); + assert.equal(sent.at(-1)[0], 'update'); + assert.match(sent.at(-1)[1].content, /Opened ticket/); +});