diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index f5a3a6a8..2c8dc5a8 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -18,6 +18,7 @@ import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveVal import { log } from '../lib/log.js'; import { every } from './schedule.js'; import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; +import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -70,6 +71,11 @@ export function normaliseKickConfig(raw = {}) { let token = { value: null, expiresAt: 0 }; +/** Forget the cached app token. For tests. */ +export function _resetToken() { + token = { value: null, expiresAt: 0 }; +} + async function appToken() { if (token.value && Date.now() < token.expiresAt - 60_000) return token.value; const body = new URLSearchParams({ @@ -78,28 +84,35 @@ async function appToken() { client_secret: config.kickClientSecret, }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`Kick token ${res.status}`); + if (!res.ok) throw tokenError('Kick token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; } -async function getChannels(slugs) { +/** + * One channels request. A network error, 5xx or 429 is retried once (see lib/pollErrors.js); the + * retry sits inside the request, so fetchIsolated only sees an error once the retry failed too. + */ +function getChannels(slugs) { const url = new URL(`${API}/channels`); for (const s of slugs) url.searchParams.append('slug', s); - const res = await fetch(url, { - headers: { - Authorization: `Bearer ${await appToken()}`, - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(10_000), + return withRetry(async () => { + const res = await fetch(url, { + headers: { + Authorization: `Bearer ${await appToken()}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }); + if (res.status === 401) { + token = { value: null, expiresAt: 0 }; // force a refresh next call + // A rejected token: it was just cleared, so one retry at once gets a fresh one. + throw Object.assign(httpError('Kick', res), { retryNow: true }); + } + if (!res.ok) throw httpError('Kick /channels', res); + return (await res.json()).data ?? []; }); - if (res.status === 401) { - token = { value: null, expiresAt: 0 }; // force a refresh next call - throw new Error('Kick 401'); - } - if (!res.ok) throw Object.assign(new Error(`Kick /channels ${res.status}`), { status: res.status }); - return (await res.json()).data ?? []; } /** @@ -183,6 +196,9 @@ export function buildPayload(channel, alert) { // --- poll loop ------------------------------------------------------- +/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */ +export const pollHealth = createPollHealth('kick-alerts'); + /** One poll; exported for tests. */ export async function tick() { if (!allClientsReady() || !config.kickEnabled) return; @@ -230,9 +246,10 @@ export async function tick() { }, }); } catch (err) { - log.error('kick-alerts', 'poll failed:', err.message); + pollHealth.failed(err); return; } + pollHealth.ok(); for (const { guildId, alert } of jobs) { // Not looked up this poll (rejected by Kick, waiting out a backoff, or over the request @@ -268,8 +285,6 @@ export async function tick() { every( 'kick-alerts', POLL_MS, - () => { - tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)); - }, + () => tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)), { firstDelayMs: 30_000 } ); diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js index 7b95ca4d..313cf8cd 100644 --- a/src/modules/lib/batchIsolation.js +++ b/src/modules/lib/batchIsolation.js @@ -31,6 +31,11 @@ // flapping): nothing is quarantined, and when no request of the poll succeeded the poll fails like // any other failed poll. If no value is known to be good yet and nothing succeeded, the poll cannot // tell: nothing is quarantined and it reports `inconclusive`. +// A canary request that fails for a network reason (lib/pollErrors.js isTransient) is not a refusal: +// with nothing succeeded the poll fails with that error, otherwise nothing is quarantined, and the +// canary is kept for the next poll. + +import { isTransient } from './pollErrors.js'; /** Requests per poll, beyond one per batch, for finding the rejected items. */ export const EXTRA_REQUESTS = 120; @@ -218,6 +223,9 @@ export async function fetchIsolated( if (succeeded === 0 && firstFailure !== null && canary) { const answer = await ask([canary], false).catch((err) => err); if (answer !== true) { + // A network failure of the canary says nothing about the values or the canary: the poll + // fails as the network failure it is, and the canary stays. + if (isTransient(answer)) throw answer; quarantine.setGood(null); throw firstFailure; } @@ -264,8 +272,10 @@ export async function fetchIsolated( // poll already fetched. const answer = await ask([canary], false).catch((err) => err); if (answer !== true) { - quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time - if (succeeded === 0) throw found[0].err; + // A network failure (after its retry) is not a refusal of the canary: it stays. + const network = isTransient(answer); + if (!network) quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time + if (succeeded === 0) throw network ? answer : found[0].err; quarantineFound = false; // flapping: some requests answer, some are refused } } else if (succeeded === 0) { diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js new file mode 100644 index 00000000..e45633f3 --- /dev/null +++ b/src/modules/lib/pollErrors.js @@ -0,0 +1,248 @@ +// Transient errors in the shared live-alert polls (Twitch and Kick). +// +// A poll asks a platform for every saved streamer at once, once a minute. A network blip (a reset +// connection, a DNS hiccup, a 502) should not page the operator, so: +// - withRetry() repeats a failed request once after a short delay, for network errors, 5xx and +// 429 (honouring Retry-After). 4xx answers other than 429 are not retried: they mean the +// request or a value in it is wrong, and asking again gives the same answer. +// - createPollHealth() counts consecutive failed polls. One line (warn) after POLL_WARN_AFTER +// failures in a row, one when polling works again, and a "still failing" reminder once an hour +// while it lasts. Errors that are not network or HTTP problems (a TypeError in our own code, a +// 4xx that needs a fix such as bad credentials) are logged as errors the first time and when +// the message changes, then once an hour ("still failing after N polls"): every error line +// also feeds /health, the error counter and the error reports, so one bad secret must not +// write 1440 of them a day. +// "fetch failed", Node's generic network error, hides the reason in `err.cause`; describeError() +// puts its code (ECONNRESET, UND_ERR_CONNECT_TIMEOUT, ...) in the text. +import { log } from '../../lib/log.js'; + +/** Failed polls in a row before one warning is logged. */ +export const POLL_WARN_AFTER = 3; + +/** How often a failure that goes on is logged again. */ +export const STILL_FAILING_MS = 60 * 60_000; + +/** Retry settings. An object so a test can shorten the wait; do not change it elsewhere. */ +export const pollRetry = { + delayMs: 3000, + /** A longer Retry-After than this is not waited for: the poll fails and the next one tries again. */ + maxWaitMs: 15_000, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +}; + +const NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', + 'ENETUNREACH', + 'ENETDOWN', + 'EHOSTUNREACH', + 'EHOSTDOWN', + 'ERR_SSL_WRONG_VERSION_NUMBER', + // undici: connection and timeout problems only. The others (UND_ERR_INVALID_ARG, ...) are + // faults in how we call it and must stay errors. + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_CLOSED', + 'UND_ERR_ABORTED', + 'UND_ERR_REQ_RETRY', +]); + +/** The error, its causes, and the members of an AggregateError (a dual-stack connect failure). */ +function chain(err) { + const out = []; + const queue = [err]; + while (queue.length > 0 && out.length < 8) { + const e = queue.shift(); + if (!e || typeof e !== 'object' || out.includes(e)) continue; + out.push(e); + queue.push(e.cause); + if (Array.isArray(e.errors)) queue.push(...e.errors); + } + return out; +} + +const isTimeoutName = (e) => e?.name === 'TimeoutError' || e?.name === 'AbortError'; + +/** The first string error code in the chain (ECONNRESET, UND_ERR_SOCKET, ...), or the timeout name. */ +export function causeCode(err) { + const items = chain(err); + for (const e of items) if (typeof e.code === 'string' && e.code) return e.code; + const named = items.find(isTimeoutName); + return named ? named.name : null; +} + +/** A failed request that never got an HTTP answer: connection, DNS, TLS, timeout, broken body. */ +export function isNetworkError(err) { + if (!err || typeof err !== 'object') return false; + if (typeof err.status === 'number') return false; // we got an answer + const items = chain(err); + if (items.some(isTimeoutName)) return true; + // undici wraps every dispatcher error in TypeError('fetch failed', { cause }): when a cause + // carries a code, the code decides (UND_ERR_INVALID_ARG is our fault, ECONNRESET is the network). + const codes = items.map((e) => e.code).filter((c) => typeof c === 'string' && c); + if (codes.length > 0) return codes.some((c) => NETWORK_CODES.has(c)); + // No code anywhere: the generic wrapper alone counts as a network failure. + return items.some( + (e) => e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated') + ); +} + +/** 5xx and 429: the platform is busy or down, try again later. Other statuses are not transient. */ +export function isTransientStatus(status) { + return typeof status === 'number' && (status >= 500 || status === 429); +} + +/** Retry once, wait a moment, and count on the next poll: network errors, 5xx and 429. */ +export function isTransient(err) { + return isNetworkError(err) || isTransientStatus(err?.status); +} + +function retryAfterMs(res) { + const header = res.headers?.get?.('retry-after'); + if (header) { + const seconds = Number(header); + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); + const date = Date.parse(header); + if (Number.isFinite(date)) return Math.max(0, date - Date.now()); + } + // Twitch states when its rate limit resets, in epoch seconds. That says nothing about an + // outage, so it is only read for a 429. + if (res.status === 429) { + const reset = Number(res.headers?.get?.('ratelimit-reset')); + if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + } + return null; +} + +/** + * The error for a non-OK answer: `${label} ${status}`, with `status`, and `retryAfterMs` when the + * platform said how long to wait (429 and 503 only). The message holds no URL, so no credentials. + */ +export function httpError(label, res) { + const err = new Error(`${label} ${res.status}`); + err.status = res.status; + if (res.status === 429 || res.status === 503) { + const wait = retryAfterMs(res); + if (wait !== null) err.retryAfterMs = wait; + } + return err; +} + +/** + * The error for a non-OK answer of the token endpoint. Its status is kept only when it is + * transient (`httpStatus` always holds it): a refused token is a fault of the request, never of + * one stored value, so batchIsolation's "rejected value" statuses (400, 404, 422) must not see it. + */ +export function tokenError(label, res) { + const err = httpError(label, res); + err.httpStatus = res.status; + if (!isTransientStatus(res.status)) delete err.status; + return err; +} + +/** + * Run `fn`; if it fails with a transient error, wait a moment and run it once more. An error with + * `retryNow` (a 401 whose token was just cleared) is retried at once. + */ +export async function withRetry(fn) { + try { + return await fn(); + } catch (err) { + if (!isTransient(err) && !err?.retryNow) throw err; + const wait = err.retryNow ? 0 : (err.retryAfterMs ?? pollRetry.delayMs); + if (wait > pollRetry.maxWaitMs) throw err; + await pollRetry.sleep(wait); + return await fn(); + } +} + +/** Short text for a log line: the message, plus the cause code when the message hides it. */ +export function describeError(err) { + const message = String(err?.message ?? err) + .replace(/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi, '') // never a URL, it may carry credentials + .slice(0, 200); + const code = causeCode(err); + return code && !message.includes(code) ? `${message} (${code})` : message; +} + +/** + * Counts consecutive failed polls of one platform. + * @param {string} scope the log scope, e.g. 'twitch-alerts' + * @param {{ warnAfter?: number, repeatMs?: number, now?: () => number }} [o] + */ +export function createPollHealth( + scope, + { warnAfter = POLL_WARN_AFTER, repeatMs = STILL_FAILING_MS, now = () => Date.now() } = {} +) { + let failures = 0; // transient failures in a row + let warned = false; + let warnedAt = 0; + /** + * The errors that are not transient and go on, by kind (name, status, message without numbers), + * so errors that alternate, or whose message holds a changing value, are not logged every poll. + * A fixed number of kinds is kept; the oldest is dropped. + * @type {Map} + */ + const faults = new Map(); + const MAX_KINDS = 10; + const kindOf = (err) => + [err?.name, err?.httpStatus ?? err?.status ?? '', describeError(err).replace(/\d+/g, '#')].join('|'); + const faultPolls = () => [...faults.values()].reduce((sum, f) => sum + f.polls, 0); + const reset = () => { + failures = 0; + warned = false; + faults.clear(); + }; + return { + /** Transient failed polls in a row. */ + get failures() { + return failures; + }, + /** A poll failed with `err`. Logs now, later or not at all, see the file head. */ + failed(err) { + if (!isTransient(err)) { + const kind = kindOf(err); + const fault = faults.get(kind); + if (!fault) { + if (faults.size >= MAX_KINDS) faults.delete(faults.keys().next().value); + faults.set(kind, { polls: 1, loggedAt: now() }); + log.error(scope, 'poll failed:', err); // the Error itself: its stack reaches the logs + return; + } + fault.polls += 1; + if (now() - fault.loggedAt >= repeatMs) { + fault.loggedAt = now(); + log.error(scope, `poll still failing after ${fault.polls} polls: ${describeError(err)}`); + } + return; + } + failures += 1; + if (failures === warnAfter) { + warned = true; + warnedAt = now(); + log.warn( + scope, + `poll failed ${failures} times in a row (${describeError(err)}); new streams may be announced late until it works again` + ); + } else if (warned && now() - warnedAt >= repeatMs) { + warnedAt = now(); + log.warn(scope, `poll still failing after ${failures} polls (${describeError(err)})`); + } + }, + /** A poll worked. */ + ok() { + if (warned || faults.size > 0) { + log.warn(scope, `polling works again after ${failures + faultPolls()} failed polls`); + } + reset(); + }, + reset, + }; +} diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 7a1056a2..7c92499a 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -18,6 +18,7 @@ import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; +import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -66,6 +67,11 @@ export function normaliseTwitchConfig(raw = {}) { let token = { value: null, expiresAt: 0 }; +/** Forget the cached app token. For tests. */ +export function _resetToken() { + token = { value: null, expiresAt: 0 }; +} + async function appToken() { if (token.value && Date.now() < token.expiresAt - 60_000) return token.value; const body = new URLSearchParams({ @@ -74,27 +80,36 @@ async function appToken() { grant_type: 'client_credentials', }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`Twitch token ${res.status}`); + if (!res.ok) throw tokenError('Twitch token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; } -async function helix(path, params) { +/** + * One Helix request. A network error, 5xx or 429 is retried once (see lib/pollErrors.js); the + * retry sits inside the request, so fetchIsolated only sees an error once the retry failed too. + * `retry: false` for requests that only decorate an announcement. + */ +function helix(path, params, { retry = true } = {}) { const url = new URL(HELIX + path); for (const [k, vals] of Object.entries(params)) { for (const v of [].concat(vals)) url.searchParams.append(k, v); } - const res = await fetch(url, { - headers: { 'Client-Id': config.twitchClientId, Authorization: `Bearer ${await appToken()}` }, - signal: AbortSignal.timeout(10_000), - }); - if (res.status === 401) { - token = { value: null, expiresAt: 0 }; // force refresh next call - throw new Error('Twitch 401'); - } - if (!res.ok) throw Object.assign(new Error(`Twitch ${path} ${res.status}`), { status: res.status }); - return (await res.json()).data ?? []; + const once = async () => { + const res = await fetch(url, { + headers: { 'Client-Id': config.twitchClientId, Authorization: `Bearer ${await appToken()}` }, + signal: AbortSignal.timeout(10_000), + }); + if (res.status === 401) { + token = { value: null, expiresAt: 0 }; // force refresh next call + // A rejected token: it was just cleared, so one retry at once gets a fresh one. + throw Object.assign(httpError('Twitch', res), { retryNow: true }); + } + if (!res.ok) throw httpError(`Twitch ${path}`, res); + return (await res.json()).data ?? []; + }; + return retry ? withRetry(once) : once(); } /** @@ -128,9 +143,12 @@ export async function fetchLiveStreams(logins, { report, groupOf } = {}) { export async function fetchUsers(logins) { const out = new Map(); try { - const { results } = await fetchIsolated(logins, 100, (batch) => helix('/users', { login: batch }), { - quarantine: createQuarantine(), - }); + const { results } = await fetchIsolated( + logins, + 100, + (batch) => helix('/users', { login: batch }, { retry: false }), + { quarantine: createQuarantine() } + ); for (const u of results) out.set(u.login.toLowerCase(), u); } catch (err) { log.debug('twitch-alerts', 'user lookup failed:', err.message); @@ -185,6 +203,9 @@ export function buildPayload(stream, user, alert) { // --- poll loop ------------------------------------------------------- +/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */ +export const pollHealth = createPollHealth('twitch-alerts'); + /** One poll; exported for tests. */ export async function tick() { if (!allClientsReady() || !config.twitchEnabled) return; @@ -234,9 +255,10 @@ export async function tick() { }); users = await fetchUsers([...streams.keys()]); } catch (err) { - log.error('twitch-alerts', 'poll failed:', err.message); + pollHealth.failed(err); return; } + pollHealth.ok(); for (const { guildId, alert } of jobs) { // Not looked up this poll (rejected by Twitch, waiting out a backoff, or over the request @@ -272,8 +294,6 @@ export async function tick() { every( 'twitch-alerts', POLL_MS, - () => { - tick().catch((err) => log.error('twitch-alerts', 'tick failed:', err.message)); - }, + () => tick().catch((err) => log.error('twitch-alerts', 'tick failed:', err.message)), { firstDelayMs: 25_000 } ); diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js new file mode 100644 index 00000000..89521c23 --- /dev/null +++ b/test/liveAlertsPollErrors.test.js @@ -0,0 +1,920 @@ +// A network blip in the Twitch or Kick poll must not reach the operator: it is retried once, and a +// run of failed polls is one warning, with the cause code in it. +process.env.TWITCH_CLIENT_ID = 'tid'; +process.env.TWITCH_CLIENT_SECRET = 'tsecret'; +process.env.KICK_CLIENT_ID = 'kid'; +process.env.KICK_CLIENT_SECRET = 'ksecret'; +await import('./helpers/tmpDb.js'); +const { default: test } = await import('node:test'); +const { default: assert } = await import('node:assert/strict'); +const { runtime } = await import('../src/runtime.js'); +const { log } = await import('../src/lib/log.js'); +const { setGuildModule } = await import('../src/db/modules.js'); +const pollErrors = await import('../src/modules/lib/pollErrors.js'); +const { + causeCode, + describeError, + isNetworkError, + isTransient, + httpError, + withRetry, + createPollHealth, + tokenError, + pollRetry, +} = pollErrors; +const { startSchedules, stopSchedules } = await import('../src/modules/schedule.js'); +const { createQuarantine, fetchIsolated } = await import('../src/modules/lib/batchIsolation.js'); +const twitch = await import('../src/modules/twitchAlerts.js'); +const kick = await import('../src/modules/kickAlerts.js'); + +const CH_A = '100000000000000001'; +const GUILD = '900000000000000601'; + +/** What undici throws for a dropped connection: the reason is in `cause`. */ +const fetchFailed = (code = 'ECONNRESET') => + new TypeError('fetch failed', { cause: Object.assign(new Error(`read ${code}`), { code }) }); +const fakeResponse = (status, headers = {}) => ({ + status, + ok: status >= 200 && status < 300, + headers: new Headers(headers), +}); + +function recordSleep(t) { + const waits = []; + t.mock.method(pollRetry, 'sleep', async (ms) => void waits.push(ms)); + return waits; +} + +// --- helper --------------------------------------------------------------------------------- + +test('describeError: puts the cause code of "fetch failed" in the text', () => { + assert.equal(describeError(fetchFailed('ECONNRESET')), 'fetch failed (ECONNRESET)'); + assert.equal( + describeError(fetchFailed('UND_ERR_CONNECT_TIMEOUT')), + 'fetch failed (UND_ERR_CONNECT_TIMEOUT)' + ); + assert.equal(describeError(fetchFailed('ENOTFOUND')), 'fetch failed (ENOTFOUND)'); + // dual-stack connect failures hide the code one level further down, in an AggregateError + const aggregate = new TypeError('fetch failed', { + cause: new AggregateError([Object.assign(new Error('x'), { code: 'ETIMEDOUT' })]), + }); + assert.equal(describeError(aggregate), 'fetch failed (ETIMEDOUT)'); + assert.equal(causeCode(new Error('plain')), null); +}); + +test('describeError: a timeout names itself, and no URL or credential gets into the text', () => { + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + assert.match(describeError(timeout), /\(TimeoutError\)$/); + const text = describeError(new Error('request to https://user:hunter2@api.example.com/x?token=abc failed')); + assert.doesNotMatch(text, /hunter2|token=abc|https:/); + assert.equal(describeError(httpError('Twitch /streams', fakeResponse(503))), 'Twitch /streams 503'); +}); + +test('isTransient: network errors, 5xx and 429 only', () => { + assert.equal(isNetworkError(fetchFailed()), true); + assert.equal(isNetworkError(new DOMException('x', 'TimeoutError')), true); + assert.equal(isNetworkError(new TypeError("Cannot read properties of undefined (reading 'data')")), false); + assert.equal(isNetworkError(new SyntaxError('Unexpected token < in JSON')), false); + for (const status of [500, 502, 503, 504, 429]) { + assert.equal(isTransient(httpError('x', fakeResponse(status))), true, String(status)); + } + for (const status of [400, 401, 403, 404, 422]) { + assert.equal(isTransient(httpError('x', fakeResponse(status))), false, String(status)); + } +}); + +test('httpError: reads Retry-After for 429 (seconds or a date) and ignores it elsewhere', () => { + assert.equal(httpError('x', fakeResponse(429, { 'retry-after': '2' })).retryAfterMs, 2000); + const at = new Date(Date.now() + 5000).toUTCString(); + const wait = httpError('x', fakeResponse(429, { 'retry-after': at })).retryAfterMs; + assert.ok(wait > 2000 && wait <= 5000, String(wait)); + assert.equal(httpError('x', fakeResponse(429)).retryAfterMs, undefined); + assert.equal(httpError('x', fakeResponse(400, { 'retry-after': '2' })).retryAfterMs, undefined); +}); + +test('withRetry: repeats once for a transient error, after the configured delay', async (t) => { + const waits = recordSleep(t); + let calls = 0; + const out = await withRetry(async () => { + calls += 1; + if (calls === 1) throw fetchFailed(); + return 'ok'; + }); + assert.equal(out, 'ok'); + assert.equal(calls, 2); + assert.deepEqual(waits, [pollRetry.delayMs]); +}); + +test('withRetry: a second failure is thrown, not retried again', async (t) => { + recordSleep(t); + let calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw httpError('x', fakeResponse(503)); + }), + /x 503/ + ); + assert.equal(calls, 2); +}); + +test('withRetry: 429 waits for Retry-After; a wait that is too long is not waited for', async (t) => { + const waits = recordSleep(t); + let calls = 0; + await withRetry(async () => { + calls += 1; + if (calls === 1) throw httpError('x', fakeResponse(429, { 'retry-after': '4' })); + }); + assert.deepEqual(waits, [4000]); + + waits.length = 0; + calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw httpError('x', fakeResponse(429, { 'retry-after': '3600' })); + }), + /x 429/ + ); + assert.equal(calls, 1); + assert.deepEqual(waits, []); +}); + +test('withRetry: 4xx answers and programming errors are not retried', async (t) => { + const waits = recordSleep(t); + for (const err of [ + httpError('x', fakeResponse(400)), + httpError('x', fakeResponse(401)), + httpError('x', fakeResponse(403)), + new TypeError("Cannot read properties of undefined (reading 'data')"), + ]) { + let calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw err; + }), + (e) => e === err + ); + assert.equal(calls, 1); + } + assert.deepEqual(waits, []); +}); + +test('createPollHealth: warns once after 3 failures in a row, once when it works again', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + const health = createPollHealth('x-alerts'); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 0); + health.failed(fetchFailed('ETIMEDOUT')); + assert.equal(warn.mock.callCount(), 1); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 1, 'silent while it keeps failing'); + health.ok(); + assert.equal(warn.mock.callCount(), 2); + assert.match(warn.mock.calls[1].arguments[1], /works again after 5 failed polls/); + health.ok(); + assert.equal(warn.mock.callCount(), 2, 'a working poll is not logged again'); + assert.equal(error.mock.callCount(), 0); +}); + +test('createPollHealth: a run of failures that ends before the third says nothing', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const health = createPollHealth('x-alerts'); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + health.ok(); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 0); +}); + +test('isNetworkError: only the connection and timeout codes of undici count', () => { + const coded = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error('x'), { code }) }); + for (const code of [ + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + ]) { + assert.equal(isNetworkError(coded(code)), true, code); + } + for (const code of ['UND_ERR_INVALID_ARG', 'UND_ERR_NOT_SUPPORTED', 'UND_ERR_INVALID_RETURN_VALUE']) { + const err = Object.assign(new Error('bad call'), { code }); + assert.equal(isNetworkError(err), false, code); + assert.equal(isTransient(err), false, code); + } +}); + +test('isNetworkError: a fetch failure with a cause is classified by the cause code', () => { + const wrapped = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error('x'), { code }) }); + for (const code of ['ECONNRESET', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'ENOTFOUND']) { + assert.equal(isNetworkError(wrapped(code)), true, code); + assert.equal(isTransient(wrapped(code)), true, code); + } + for (const code of ['UND_ERR_INVALID_ARG', 'UND_ERR_NOT_SUPPORTED', 'UND_ERR_INVALID_RETURN_VALUE']) { + assert.equal(isNetworkError(wrapped(code)), false, code); + assert.equal(isTransient(wrapped(code)), false, code); + } + // the wrapper alone, or with a cause that has no code, still counts as a network failure + assert.equal(isNetworkError(new TypeError('fetch failed')), true); + assert.equal(isNetworkError(new TypeError('fetch failed', { cause: new Error('no code') })), true); + assert.equal(isNetworkError(new TypeError('Cannot read properties of undefined')), false); +}); + +test('httpError: Ratelimit-Reset is only the wait for a 429, never for a 503', () => { + const reset = String(Math.floor(Date.now() / 1000) + 8); + const wait = httpError('x', fakeResponse(429, { 'ratelimit-reset': reset })).retryAfterMs; + assert.ok(wait > 4000 && wait <= 8000, String(wait)); + assert.equal(httpError('x', fakeResponse(503, { 'ratelimit-reset': reset })).retryAfterMs, undefined); + assert.equal(httpError('x', fakeResponse(503, { 'retry-after': '2' })).retryAfterMs, 2000); + const past = new Date(Date.now() - 60_000).toUTCString(); + assert.equal(httpError('x', fakeResponse(503, { 'retry-after': past })).retryAfterMs, 0); +}); + +test('tokenError: a refused token never carries a status that batch isolation reads as a value fault', () => { + for (const status of [400, 401, 403, 404, 422]) { + const err = tokenError('Twitch token', fakeResponse(status)); + assert.equal(err.status, undefined, String(status)); + assert.equal(err.httpStatus, status); + assert.equal(err.message, `Twitch token ${status}`); + assert.equal(isTransient(err), false); + } + for (const status of [429, 500, 503]) { + const err = tokenError('Twitch token', fakeResponse(status)); + assert.equal(err.status, status); + assert.equal(isTransient(err), true); + } +}); + +test('withRetry: an error marked retryNow is retried at once, but only once', async (t) => { + const waits = recordSleep(t); + let calls = 0; + await withRetry(async () => { + calls += 1; + if (calls === 1) throw Object.assign(httpError('x', fakeResponse(401)), { retryNow: true }); + }); + assert.equal(calls, 2); + assert.deepEqual(waits, [0]); + calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw Object.assign(httpError('x', fakeResponse(401)), { retryNow: true }); + }), + /x 401/ + ); + assert.equal(calls, 2); +}); + +test('createPollHealth: an error that is not transient is logged once, again when it changes, then hourly', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + let now = 1_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + const denied = () => httpError('Twitch /streams', fakeResponse(403)); + const bug = new TypeError("Cannot read properties of undefined (reading 'x')"); + health.failed(bug); + assert.equal(error.mock.callCount(), 1); + assert.equal(error.mock.calls[0].arguments[2], bug, 'the Error itself, so its stack is logged'); + for (let i = 0; i < 100; i += 1) { + now += 60_000; + health.failed(bug); + } + assert.equal(error.mock.callCount(), 2, 'silent for the first hour, then one reminder'); + assert.match(error.mock.calls[1].arguments[1], /still failing after \d+ polls: Cannot read properties/); + health.failed(denied()); // a different message: logged at once + assert.equal(error.mock.callCount(), 3); + now += 61 * 60_000; + health.failed(denied()); + health.failed(denied()); + assert.equal(error.mock.callCount(), 4); + assert.match(error.mock.calls[3].arguments[1], /still failing after 2 polls: Twitch \/streams 403/); + assert.equal(health.failures, 0, 'not a network failure'); + health.ok(); + assert.equal(warn.mock.callCount(), 1); + assert.match(warn.mock.calls[0].arguments[1], /works again after 104 failed polls/); + health.ok(); + assert.equal(warn.mock.callCount(), 1); +}); + +test('createPollHealth: alternating errors, or a message with a changing number, are not logged on every poll', (t) => { + const error = t.mock.method(log, 'error', () => {}); + let now = 9_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + const a = () => httpError('Twitch /streams', fakeResponse(403)); + const b = () => httpError('Twitch token', fakeResponse(401)); + for (let i = 0; i < 20; i += 1) { + health.failed(a()); + health.failed(b()); + now += 60_000; + } + assert.equal(error.mock.callCount(), 2, 'one line for each of the two kinds'); + for (let i = 0; i < 30; i += 1) { + health.failed(new RangeError(`index ${i} is out of range 0..${100 + i}`)); + now += 60_000; + } + assert.equal(error.mock.callCount(), 3, 'the number in the message does not make a new kind'); + // a different status is a different kind + health.failed(httpError('Twitch /streams', fakeResponse(404))); + assert.equal(error.mock.callCount(), 4); +}); + +test('createPollHealth: a long network outage gets a reminder every hour', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + let now = 5_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + for (let i = 0; i < 130; i += 1) { + health.failed(fetchFailed('ENOTFOUND')); + now += 60_000; + } + // 130 minutes: the warning at the 3rd failure, then one reminder per hour + assert.equal(warn.mock.callCount(), 3); + assert.match(warn.mock.calls[0].arguments[1], /3 times in a row/); + assert.match( + warn.mock.calls[1].arguments[1], + /still failing after \d+ polls \(fetch failed \(ENOTFOUND\)\)/ + ); + assert.match(warn.mock.calls[2].arguments[1], /still failing after \d+ polls/); +}); + +// --- ticks ---------------------------------------------------------------------------------- + +function installClient(sent) { + const channel = { + isTextBased: () => true, + permissionsFor: () => ({ has: () => true }), + send: async (payload) => { + sent.push(payload); + return { id: `msg-${sent.length}` }; + }, + }; + const guild = { + id: GUILD, + channels: { cache: new Map([[CH_A, channel]]), fetch: async () => channel }, + members: { me: {} }, + }; + runtime.client = { isReady: () => true, guilds: { cache: new Map([[GUILD, guild]]) } }; + guild.client = runtime.client; +} + +const tokenOk = () => Response.json({ access_token: 't', expires_in: 3600 }); + +/** + * Run `body` with a fake global fetch. `answer(url, n)` gets the URL and how many requests the + * platform's data endpoints have seen and returns a Response or throws. `token(n)` answers the n-th + * request of the token endpoint (default: it works). + */ +async function withFakeFetch(dataHost, answer, body, { token } = {}) { + const realFetch = globalThis.fetch; + const seen = { data: 0, token: 0 }; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.pathname.includes('token')) { + seen.token += 1; + return token ? token(seen.token) : tokenOk(); + } + assert.equal(u.hostname, dataHost); + seen.data += 1; + return answer(u, seen.data); + }; + try { + await body(seen); + } finally { + globalThis.fetch = realFetch; + } +} + +const PLATFORMS = [ + { + name: 'twitch', + mod: twitch, + scope: 'twitch-alerts', + host: 'api.twitch.tv', + save: (value) => + setGuildModule(GUILD, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: value, channelId: CH_A }] }, + }), + liveAnswer: (u, login) => + Response.json({ + data: u.pathname.endsWith('/users') + ? [] + : [ + { + id: `s-${login}`, + user_login: login, + user_name: login, + title: 'hi', + game_name: 'g', + viewer_count: 3, + started_at: '2026-11-16T00:00:00Z', + thumbnail_url: '', + }, + ], + }), + }, + { + name: 'kick', + mod: kick, + scope: 'kick-alerts', + host: 'api.kick.com', + save: (value) => + setGuildModule(GUILD, 'kick-alerts', { + enabled: true, + config: { alerts: [{ slug: value, channelId: CH_A }] }, + }), + liveAnswer: (u, slug) => + Response.json({ + data: [ + { + slug, + stream_title: 'hi', + category: { name: 'g' }, + stream: { + is_live: true, + start_time: '2026-11-16T00:00:00Z', + viewer_count: 2, + url: `https://kick.com/${slug}`, + }, + }, + ], + }), + }, +]; + +for (const p of PLATFORMS) { + const prep = async (t, value) => { + const sent = []; + installClient(sent); + p.mod.pollHealth.reset(); + p.mod._resetToken(); + const waits = recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await p.save(value); + return { sent, waits, warn, error }; + }; + + test(`${p.name} tick: one network blip is retried silently and the stream is announced`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'blipone'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? Promise.reject(fetchFailed('ECONNRESET')) : p.liveAnswer(u, 'blipone')), + async (seen) => { + await p.mod.tick(); + assert.ok(seen.data >= 2, 'the request was repeated'); + } + ); + assert.equal(sent.length, 1, 'announced as usual'); + assert.deepEqual(waits, [pollRetry.delayMs]); + assert.equal(warn.mock.callCount(), 0); + assert.equal(error.mock.callCount(), 0); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + test(`${p.name} tick: a 503 is retried once too`, async (t) => { + const { sent, warn, error } = await prep(t, 'fivehundred'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? new Response('{}', { status: 503 }) : p.liveAnswer(u, 'fivehundred')), + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: 429 waits for Retry-After before the retry`, async (t) => { + const { sent, waits } = await prep(t, 'slowdown'); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? new Response('{}', { status: 429, headers: { 'Retry-After': '3' } }) + : p.liveAnswer(u, 'slowdown'), + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(waits, [3000]); + assert.equal(sent.length, 1); + }); + + test(`${p.name} tick: 3 failed polls in a row are one warning with the cause code, recovery is one more line`, async (t) => { + const { sent, warn, error } = await prep(t, 'flaky'); + let down = true; + await withFakeFetch( + p.host, + (u) => (down ? Promise.reject(fetchFailed('ECONNRESET')) : p.liveAnswer(u, 'flaky')), + async (seen) => { + await p.mod.tick(); + await p.mod.tick(); + assert.equal(seen.data, 4, 'each failed poll tried twice'); + assert.equal(warn.mock.callCount(), 0, 'one or two failed polls are not logged'); + await p.mod.tick(); + assert.equal(warn.mock.callCount(), 1); + const [scope, line] = warn.mock.calls[0].arguments; + assert.equal(scope, p.scope); + assert.match(line, /3 times in a row/); + assert.match(line, /fetch failed \(ECONNRESET\)/); + assert.doesNotMatch(line, /https?:|tsecret|ksecret|Bearer/); + await p.mod.tick(); + await p.mod.tick(); + assert.equal(warn.mock.callCount(), 1, 'no repeat while it stays down'); + assert.equal(sent.length, 0); + + down = false; + await p.mod.tick(); + } + ); + assert.equal(warn.mock.callCount(), 2); + assert.match(warn.mock.calls[1].arguments[1], /works again after 5 failed polls/); + assert.equal(sent.length, 1, 'the recovered poll announces as usual'); + assert.equal(error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a programming error is still an error, and is not retried`, async (t) => { + const { waits, warn, error } = await prep(t, 'buggy'); + await withFakeFetch( + p.host, + () => Promise.reject(new TypeError("Cannot read properties of undefined (reading 'x')")), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 1, 'not retried'); + } + ); + assert.equal(error.mock.callCount(), 1); + assert.equal(error.mock.calls[0].arguments[0], p.scope); + assert.match(error.mock.calls[0].arguments[2].message, /Cannot read properties/); + assert.ok(error.mock.calls[0].arguments[2].stack, 'the Error itself is logged, with its stack'); + assert.equal(warn.mock.callCount(), 0); + assert.deepEqual(waits, []); + assert.equal(p.mod.pollHealth.failures, 0, 'does not count as a network failure'); + }); + + test(`${p.name} tick: a 4xx answer is not retried and still logged as an error`, async (t) => { + const { waits, warn, error } = await prep(t, 'denied'); + await withFakeFetch( + p.host, + () => new Response('{}', { status: 403 }), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 1); + } + ); + assert.equal(error.mock.callCount(), 1); + assert.match(error.mock.calls[0].arguments[2].message, /403/); + assert.equal(warn.mock.callCount(), 0); + assert.deepEqual(waits, []); + }); + + const tokenBlips = [ + ['a 503', () => new Response('{}', { status: 503 })], + ['a network blip', () => Promise.reject(fetchFailed('ECONNRESET'))], + ]; + for (const [i, [what, first]] of tokenBlips.entries()) { + test(`${p.name} tick: the token endpoint answering with ${what} is retried silently`, async (t) => { + const { sent, waits, warn, error } = await prep(t, `tokenblip${i}`); + await withFakeFetch( + p.host, + (u) => p.liveAnswer(u, `tokenblip${i}`), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 2, 'the token request was repeated'); + }, + { token: (n) => (n === 1 ? first() : tokenOk()) } + ); + assert.equal(sent.length, 1); + assert.deepEqual(waits, [pollRetry.delayMs]); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + } + + test(`${p.name} tick: a token endpoint that refuses with 400 is one error, with no search and no data request`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'badclient'); + await withFakeFetch( + p.host, + (u) => p.liveAnswer(u, 'badclient'), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 1, 'one token request, not one per isolated batch'); + assert.equal(seen.data, 0); + }, + { token: () => new Response('{"error":"invalid client"}', { status: 400 }) } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 1); + assert.match(error.mock.calls[0].arguments[2].message, /token 400/); + assert.equal(warn.mock.callCount(), 0, 'no "every request was refused" warning'); + assert.deepEqual(waits, []); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + const retryAfters = [ + ['seconds', '2', 2000], + ['a date that has passed', new Date(Date.now() - 60_000).toUTCString(), 0], + ]; + for (const [i, [what, header, expected]] of retryAfters.entries()) { + test(`${p.name} tick: a 503 honours Retry-After given as ${what}`, async (t) => { + const { sent, waits, warn, error } = await prep(t, `retryafter${i}`); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? new Response('{}', { status: 503, headers: { 'Retry-After': header } }) + : p.liveAnswer(u, `retryafter${i}`), + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(waits, [expected]); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + } + + test(`${p.name} tick: a request timeout is retried silently`, async (t) => { + const { sent, warn, error } = await prep(t, 'timedout'); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? Promise.reject(new DOMException('The operation was aborted due to timeout', 'TimeoutError')) + : p.liveAnswer(u, 'timedout'), + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a rejected token (401) is refreshed and the request retried at once, silently`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'rotated'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? new Response('{}', { status: 401 }) : p.liveAnswer(u, 'rotated')), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 2, 'a fresh token for the retry'); + } + ); + assert.deepEqual(waits, [0]); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a 401 that stays is an error`, async (t) => { + const { sent, error } = await prep(t, 'norights'); + await withFakeFetch( + p.host, + () => new Response('{}', { status: 401 }), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 2, 'tried once more, not more'); + } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 1); + }); + + test(`${p.name} tick: bad credentials are one error line for many polls, then one recovery line`, async (t) => { + const { sent, warn, error } = await prep(t, 'badsecret'); + let denied = true; + await withFakeFetch( + p.host, + (u) => (denied ? new Response('{}', { status: 403 }) : p.liveAnswer(u, 'badsecret')), + async () => { + for (let i = 0; i < 5; i += 1) await p.mod.tick(); + assert.equal(error.mock.callCount(), 1, 'logged once, not on every poll'); + denied = false; + await p.mod.tick(); + } + ); + assert.equal(warn.mock.callCount(), 1); + assert.match(warn.mock.calls[0].arguments[1], /works again after 5 failed polls/); + assert.equal(sent.length, 1); + }); +} + +test('the poll callbacks hand their tick to the scheduler: a tick that is still running is not started twice', async (t) => { + const sent = []; + installClient(sent); + recordSleep(t); + t.mock.method(log, 'warn', () => {}); + t.mock.method(log, 'error', () => {}); + for (const p of PLATFORMS) { + p.mod.pollHealth.reset(); + p.mod._resetToken(); + await p.save('slowtick'); + } + const hits = {}; + const release = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (url) => { + const u = new URL(String(url)); + if (u.pathname.includes('token')) return Promise.resolve(tokenOk()); + hits[u.hostname] = (hits[u.hostname] ?? 0) + 1; + const p = PLATFORMS.find((x) => x.host === u.hostname); + return new Promise((resolve) => release.push(() => resolve(p.liveAnswer(u, 'slowtick')))); + }; + const flush = async () => { + for (let i = 0; i < 30; i += 1) await new Promise((resolve) => setImmediate(resolve)); + }; + t.mock.timers.enable({ apis: ['setTimeout', 'setInterval'] }); + try { + startSchedules(); + t.mock.timers.tick(30_000); // the first ticks (Twitch 25 s, Kick 30 s), now waiting on the network + await flush(); + t.mock.timers.tick(60_000); // the next interval comes while they are still waiting + await flush(); + assert.deepEqual(hits, { 'api.twitch.tv': 1, 'api.kick.com': 1 }, 'no second tick on top of the first'); + for (let i = 0; i < 10 && release.length > 0; i += 1) { + for (const go of release.splice(0)) go(); + await flush(); + } + await stopSchedules(); + } finally { + globalThis.fetch = realFetch; + } +}); + +// --- composition with the batch isolation of rejected values --------------------------------- + +for (const p of PLATFORMS) { + // a different rejected value per test: a quarantined one is left out of the next poll + const poisonFor = (n) => (p.name === 'twitch' ? '_poison' : '-poison') + n; + const names = (u) => + p.name === 'twitch' + ? u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login') + : u.searchParams.getAll('slug'); + const saveTwo = (value, poison) => + p.name === 'twitch' + ? setGuildModule(GUILD, 'twitch-alerts', { + enabled: true, + config: { + alerts: [ + { login: poison, channelId: CH_A }, + { login: value, channelId: CH_A }, + ], + }, + }) + : setGuildModule(GUILD, 'kick-alerts', { + enabled: true, + config: { + alerts: [ + { slug: poison, channelId: CH_A }, + { slug: value, channelId: CH_A }, + ], + }, + }); + + test(`${p.name} tick: a blip and a rejected value in the same poll: retried, isolated, victim announced`, async (t) => { + const sent = []; + installClient(sent); + const poison = poisonFor('a'); + p.mod.pollHealth.reset(); + p.mod._resetToken(); + recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await saveTwo('victimblip', poison); + await withFakeFetch( + p.host, + (u, n) => { + if (n === 1) return Promise.reject(fetchFailed('ECONNRESET')); + if (names(u).includes(poison)) return new Response('{}', { status: 400 }); + return p.liveAnswer(u, 'victimblip'); + }, + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1, 'the victim is announced'); + assert.equal(error.mock.callCount(), 0); + assert.equal(warn.mock.callCount(), 1, 'only the warning about the rejected value'); + assert.match(warn.mock.calls[0].arguments[1], /rejected 1 of the/); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + test(`${p.name} tick: a network error while values are being isolated fails the poll as a network failure, nothing is quarantined`, async (t) => { + const sent = []; + installClient(sent); + const poison = poisonFor('b'); + p.mod.pollHealth.reset(); + p.mod._resetToken(); + recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await saveTwo('victimdown', poison); + const quarantine = p.name === 'twitch' ? p.mod.rejectedLogins : p.mod.rejectedSlugs; + const before = quarantine.size; + const asked = []; + await withFakeFetch( + p.host, + (u) => { + asked.push(names(u)); + // the value known to be good (the canary) is still answered, so the search starts + if (names(u).length === 1 && names(u)[0] === quarantine.lastGood) + return p.liveAnswer(u, quarantine.lastGood); + if (names(u).length === 2) return new Response('{}', { status: 400 }); + if (names(u).includes(poison)) return new Response('{}', { status: 400 }); + return Promise.reject(fetchFailed('UND_ERR_CONNECT_TIMEOUT')); + }, + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(asked[0].sort(), [poison, 'victimdown'].sort(), 'both values were asked together first'); + assert.ok( + asked.some((names) => names.length === 1 && names[0] === poison), + 'the search got to the rejected value' + ); + assert.ok( + asked.some((names) => names.length === 1 && names[0] === 'victimdown'), + 'and was cut short by the network error' + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 0); + assert.equal(warn.mock.callCount(), 0, 'one failed poll is not logged'); + assert.equal(p.mod.pollHealth.failures, 1); + assert.equal(quarantine.size, before, 'a network error is not a value fault'); + }); +} + +test('fetchIsolated: a network failure of the canary fails the poll as that failure and keeps the canary', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const refused = (status) => Object.assign(new Error(`refused ${status}`), { status }); + await assert.rejects( + fetchIsolated( + ['a', 'b'], + 2, + async (batch) => { + if (batch[0] === 'known-good') throw fetchFailed('ECONNRESET'); + throw refused(400); + }, + { quarantine } + ), + (err) => err.message === 'fetch failed', + 'not the 400 of the first batch' + ); + assert.equal(quarantine.lastGood, 'known-good', 'the canary was not thrown away'); + assert.equal(quarantine.size, 0); +}); + +test('fetchIsolated: a network failure of the canary while the platform flaps quarantines nothing and remembers the value for the next poll', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const out = await fetchIsolated( + ['fine1', 'flaky'], + 1, + async (batch) => { + if (batch[0] === 'flaky') throw Object.assign(new Error('refused 404'), { status: 404 }); + if (batch[0] === 'known-good') throw fetchFailed('ETIMEDOUT'); + return batch; + }, + { quarantine } + ); + assert.deepEqual(out.results, ['fine1']); + assert.equal(quarantine.size, 0, 'a value is not quarantined on the word of a failing request'); + assert.ok(out.unresolved.has('flaky')); + assert.ok(quarantine.suspects.has('flaky'), 'it is batched apart from the clean values next time'); +}); + +test('twitch tick: a failed user lookup is neither retried nor logged, the stream is still announced', async (t) => { + const sent = []; + installClient(sent); + twitch.pollHealth.reset(); + const waits = recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await PLATFORMS[0].save('nousers'); + let userCalls = 0; + await withFakeFetch( + 'api.twitch.tv', + (u) => { + if (u.pathname.endsWith('/users')) { + userCalls += 1; + return Promise.reject(fetchFailed('ECONNRESET')); + } + return PLATFORMS[0].liveAnswer(u, 'nousers'); + }, + async () => { + await twitch.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(userCalls, 1); + assert.deepEqual(waits, []); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); +}); + +test.after(() => { + runtime.client = null; +});