From 56faae91b24baa6cc05264c675a2b80dec0d012a Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 11:46:39 +0100 Subject: [PATCH 1/3] fix(live-alerts): retry a failed Twitch/Kick poll once and log a run of failures once A single network blip ("fetch failed") no longer logs an error. A failed request is retried once after a short delay for network errors, 5xx and 429 (honouring Retry-After). Failed polls are counted per platform: one warning after 3 in a row, naming the cause code (e.g. ECONNRESET), and one line when polling works again. Programming errors and 4xx answers are still logged as errors. --- src/modules/kickAlerts.js | 41 +-- src/modules/lib/pollErrors.js | 176 +++++++++++++ src/modules/twitchAlerts.js | 35 ++- test/liveAlertsPollErrors.test.js | 419 ++++++++++++++++++++++++++++++ 4 files changed, 642 insertions(+), 29 deletions(-) create mode 100644 src/modules/lib/pollErrors.js create mode 100644 test/liveAlertsPollErrors.test.js diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index 45913308..631c8668 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -17,6 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; +import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -77,28 +78,31 @@ async function appToken() { client_secret: config.kickClientSecret, }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`Kick token ${res.status}`); + if (!res.ok) throw httpError('Kick token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; } -async function getChannels(slugs) { +/** One channels request; a network error, 5xx or 429 is retried once (see lib/pollErrors.js). */ +function getChannels(slugs) { const url = new URL(`${API}/channels`); for (const s of slugs) url.searchParams.append('slug', s); - const res = await fetch(url, { - headers: { - Authorization: `Bearer ${await appToken()}`, - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(10_000), + return withRetry(async () => { + const res = await fetch(url, { + headers: { + Authorization: `Bearer ${await appToken()}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }); + if (res.status === 401) { + token = { value: null, expiresAt: 0 }; // force a refresh next call + throw httpError('Kick', res); + } + if (!res.ok) throw httpError('Kick /channels', res); + return (await res.json()).data ?? []; }); - if (res.status === 401) { - token = { value: null, expiresAt: 0 }; // force a refresh next call - throw new Error('Kick 401'); - } - if (!res.ok) throw new Error(`Kick /channels ${res.status}`); - return (await res.json()).data ?? []; } /** @@ -166,7 +170,11 @@ export function buildPayload(channel, alert) { // --- poll loop ------------------------------------------------------- -async function tick() { +/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */ +export const pollHealth = createPollHealth('kick-alerts'); + +/** One poll; exported for tests. */ +export async function tick() { if (!allClientsReady() || !config.kickEnabled) return; const jobs = []; @@ -182,9 +190,10 @@ async function tick() { try { live = await fetchLiveChannels(slugs); } catch (err) { - log.error('kick-alerts', 'poll failed:', err.message); + pollHealth.failed(err); return; } + pollHealth.ok(); for (const { guildId, alert } of jobs) { const channel = live.get(alert.slug); diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js new file mode 100644 index 00000000..e008bb1d --- /dev/null +++ b/src/modules/lib/pollErrors.js @@ -0,0 +1,176 @@ +// Transient errors in the shared live-alert polls (Twitch and Kick). +// +// A poll asks a platform for every saved streamer at once, once a minute. A network blip (a reset +// connection, a DNS hiccup, a 502) should not page the operator, so: +// - withRetry() repeats a failed request once after a short delay, for network errors, 5xx and +// 429 (honouring Retry-After). 4xx answers other than 429 are not retried: they mean the +// request or a value in it is wrong, and asking again gives the same answer. +// - createPollHealth() counts consecutive failed polls. One line (warn) after POLL_WARN_AFTER +// failures in a row, one when polling works again, nothing in between. Errors that are not +// network or HTTP problems (a TypeError in our own code, a 4xx that needs a fix such as bad +// credentials) are still logged as errors every time. +// "fetch failed", Node's generic network error, hides the reason in `err.cause`; describeError() +// puts its code (ECONNRESET, UND_ERR_CONNECT_TIMEOUT, ...) in the text. +import { log } from '../../lib/log.js'; + +/** Failed polls in a row before one warning is logged. */ +export const POLL_WARN_AFTER = 3; + +/** Retry settings. An object so a test can shorten the wait; do not change it elsewhere. */ +export const pollRetry = { + delayMs: 3000, + /** A longer Retry-After than this is not waited for: the poll fails and the next one tries again. */ + maxWaitMs: 15_000, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +}; + +const NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', + 'ENETUNREACH', + 'ENETDOWN', + 'EHOSTUNREACH', + 'EHOSTDOWN', + 'ERR_SSL_WRONG_VERSION_NUMBER', +]); + +/** The error, its causes, and the members of an AggregateError (a dual-stack connect failure). */ +function chain(err) { + const out = []; + const queue = [err]; + while (queue.length > 0 && out.length < 8) { + const e = queue.shift(); + if (!e || typeof e !== 'object' || out.includes(e)) continue; + out.push(e); + queue.push(e.cause); + if (Array.isArray(e.errors)) queue.push(...e.errors); + } + return out; +} + +const isTimeoutName = (e) => e?.name === 'TimeoutError' || e?.name === 'AbortError'; + +/** The first string error code in the chain (ECONNRESET, UND_ERR_SOCKET, ...), or the timeout name. */ +export function causeCode(err) { + const items = chain(err); + for (const e of items) if (typeof e.code === 'string' && e.code) return e.code; + const named = items.find(isTimeoutName); + return named ? named.name : null; +} + +/** A failed request that never got an HTTP answer: connection, DNS, TLS, timeout, broken body. */ +export function isNetworkError(err) { + if (!err || typeof err !== 'object') return false; + if (typeof err.status === 'number') return false; // we got an answer + return chain(err).some( + (e) => + isTimeoutName(e) || + (typeof e.code === 'string' && (NETWORK_CODES.has(e.code) || e.code.startsWith('UND_ERR_'))) || + (e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')) + ); +} + +/** 5xx and 429: the platform is busy or down, try again later. Other statuses are not transient. */ +export function isTransientStatus(status) { + return typeof status === 'number' && (status >= 500 || status === 429); +} + +/** Retry once, wait a moment, and count on the next poll: network errors, 5xx and 429. */ +export function isTransient(err) { + return isNetworkError(err) || isTransientStatus(err?.status); +} + +function retryAfterMs(res) { + const header = res.headers?.get?.('retry-after'); + if (header) { + const seconds = Number(header); + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); + const date = Date.parse(header); + if (Number.isFinite(date)) return Math.max(0, date - Date.now()); + } + // Twitch states when its rate limit resets, in epoch seconds. + const reset = Number(res.headers?.get?.('ratelimit-reset')); + if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + return null; +} + +/** + * The error for a non-OK answer: `${label} ${status}`, with `status`, and `retryAfterMs` when the + * platform said how long to wait (429 and 503 only). The message holds no URL, so no credentials. + */ +export function httpError(label, res) { + const err = new Error(`${label} ${res.status}`); + err.status = res.status; + if (res.status === 429 || res.status === 503) { + const wait = retryAfterMs(res); + if (wait !== null) err.retryAfterMs = wait; + } + return err; +} + +/** Run `fn`; if it fails with a transient error, wait a moment and run it once more. */ +export async function withRetry(fn) { + try { + return await fn(); + } catch (err) { + if (!isTransient(err)) throw err; + const wait = err.retryAfterMs ?? pollRetry.delayMs; + if (wait > pollRetry.maxWaitMs) throw err; + await pollRetry.sleep(wait); + return await fn(); + } +} + +/** Short text for a log line: the message, plus the cause code when the message hides it. */ +export function describeError(err) { + const message = String(err?.message ?? err) + .replace(/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi, '') // never a URL, it may carry credentials + .slice(0, 200); + const code = causeCode(err); + return code && !message.includes(code) ? `${message} (${code})` : message; +} + +/** + * Counts consecutive failed polls of one platform. + * @param {string} scope the log scope, e.g. 'twitch-alerts' + */ +export function createPollHealth(scope, { warnAfter = POLL_WARN_AFTER } = {}) { + let failures = 0; + let warned = false; + return { + /** Failed polls in a row. */ + get failures() { + return failures; + }, + /** A poll failed with `err`. Logs now, or only after `warnAfter` in a row, see the file head. */ + failed(err) { + if (!isTransient(err)) { + log.error(scope, 'poll failed:', describeError(err)); + return; + } + failures += 1; + if (failures === warnAfter) { + warned = true; + log.warn( + scope, + `poll failed ${failures} times in a row (${describeError(err)}); new streams may be announced late until it works again` + ); + } + }, + /** A poll worked. */ + ok() { + if (warned) log.warn(scope, `polling works again after ${failures} failed polls`); + failures = 0; + warned = false; + }, + reset() { + failures = 0; + warned = false; + }, + }; +} diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 3abb2655..1b7d12dd 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -17,6 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; +import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -73,27 +74,30 @@ async function appToken() { grant_type: 'client_credentials', }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`Twitch token ${res.status}`); + if (!res.ok) throw httpError('Twitch token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; } -async function helix(path, params) { +/** One Helix request; a network error, 5xx or 429 is retried once (see lib/pollErrors.js). */ +function helix(path, params) { const url = new URL(HELIX + path); for (const [k, vals] of Object.entries(params)) { for (const v of [].concat(vals)) url.searchParams.append(k, v); } - const res = await fetch(url, { - headers: { 'Client-Id': config.twitchClientId, Authorization: `Bearer ${await appToken()}` }, - signal: AbortSignal.timeout(10_000), + return withRetry(async () => { + const res = await fetch(url, { + headers: { 'Client-Id': config.twitchClientId, Authorization: `Bearer ${await appToken()}` }, + signal: AbortSignal.timeout(10_000), + }); + if (res.status === 401) { + token = { value: null, expiresAt: 0 }; // force refresh next call + throw httpError('Twitch', res); + } + if (!res.ok) throw httpError(`Twitch ${path}`, res); + return (await res.json()).data ?? []; }); - if (res.status === 401) { - token = { value: null, expiresAt: 0 }; // force refresh next call - throw new Error('Twitch 401'); - } - if (!res.ok) throw new Error(`Twitch ${path} ${res.status}`); - return (await res.json()).data ?? []; } /** Live streams for a set of logins, keyed by lowercased login. */ @@ -166,7 +170,11 @@ export function buildPayload(stream, user, alert) { // --- poll loop ------------------------------------------------------- -async function tick() { +/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */ +export const pollHealth = createPollHealth('twitch-alerts'); + +/** One poll; exported for tests. */ +export async function tick() { if (!allClientsReady() || !config.twitchEnabled) return; // Gather every alert across enabled guilds. @@ -185,9 +193,10 @@ async function tick() { streams = await fetchLiveStreams(logins); users = await fetchUsers([...streams.keys()]); } catch (err) { - log.error('twitch-alerts', 'poll failed:', err.message); + pollHealth.failed(err); return; } + pollHealth.ok(); for (const { guildId, alert } of jobs) { const stream = streams.get(alert.login); diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js new file mode 100644 index 00000000..a55a008c --- /dev/null +++ b/test/liveAlertsPollErrors.test.js @@ -0,0 +1,419 @@ +// A network blip in the Twitch or Kick poll must not reach the operator: it is retried once, and a +// run of failed polls is one warning, with the cause code in it. +process.env.TWITCH_CLIENT_ID = 'tid'; +process.env.TWITCH_CLIENT_SECRET = 'tsecret'; +process.env.KICK_CLIENT_ID = 'kid'; +process.env.KICK_CLIENT_SECRET = 'ksecret'; +await import('./helpers/tmpDb.js'); +const { default: test } = await import('node:test'); +const { default: assert } = await import('node:assert/strict'); +const { runtime } = await import('../src/runtime.js'); +const { log } = await import('../src/lib/log.js'); +const { setGuildModule } = await import('../src/db/modules.js'); +const pollErrors = await import('../src/modules/lib/pollErrors.js'); +const { + causeCode, + describeError, + isNetworkError, + isTransient, + httpError, + withRetry, + createPollHealth, + pollRetry, +} = pollErrors; +const twitch = await import('../src/modules/twitchAlerts.js'); +const kick = await import('../src/modules/kickAlerts.js'); + +const CH_A = '100000000000000001'; +const GUILD = '900000000000000601'; + +/** What undici throws for a dropped connection: the reason is in `cause`. */ +const fetchFailed = (code = 'ECONNRESET') => + new TypeError('fetch failed', { cause: Object.assign(new Error(`read ${code}`), { code }) }); +const fakeResponse = (status, headers = {}) => ({ + status, + ok: status >= 200 && status < 300, + headers: new Headers(headers), +}); + +function recordSleep(t) { + const waits = []; + t.mock.method(pollRetry, 'sleep', async (ms) => void waits.push(ms)); + return waits; +} + +// --- helper --------------------------------------------------------------------------------- + +test('describeError: puts the cause code of "fetch failed" in the text', () => { + assert.equal(describeError(fetchFailed('ECONNRESET')), 'fetch failed (ECONNRESET)'); + assert.equal( + describeError(fetchFailed('UND_ERR_CONNECT_TIMEOUT')), + 'fetch failed (UND_ERR_CONNECT_TIMEOUT)' + ); + assert.equal(describeError(fetchFailed('ENOTFOUND')), 'fetch failed (ENOTFOUND)'); + // dual-stack connect failures hide the code one level further down, in an AggregateError + const aggregate = new TypeError('fetch failed', { + cause: new AggregateError([Object.assign(new Error('x'), { code: 'ETIMEDOUT' })]), + }); + assert.equal(describeError(aggregate), 'fetch failed (ETIMEDOUT)'); + assert.equal(causeCode(new Error('plain')), null); +}); + +test('describeError: a timeout names itself, and no URL or credential gets into the text', () => { + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + assert.match(describeError(timeout), /\(TimeoutError\)$/); + const text = describeError(new Error('request to https://user:hunter2@api.example.com/x?token=abc failed')); + assert.doesNotMatch(text, /hunter2|token=abc|https:/); + assert.equal(describeError(httpError('Twitch /streams', fakeResponse(503))), 'Twitch /streams 503'); +}); + +test('isTransient: network errors, 5xx and 429 only', () => { + assert.equal(isNetworkError(fetchFailed()), true); + assert.equal(isNetworkError(new DOMException('x', 'TimeoutError')), true); + assert.equal(isNetworkError(new TypeError("Cannot read properties of undefined (reading 'data')")), false); + assert.equal(isNetworkError(new SyntaxError('Unexpected token < in JSON')), false); + for (const status of [500, 502, 503, 504, 429]) { + assert.equal(isTransient(httpError('x', fakeResponse(status))), true, String(status)); + } + for (const status of [400, 401, 403, 404, 422]) { + assert.equal(isTransient(httpError('x', fakeResponse(status))), false, String(status)); + } +}); + +test('httpError: reads Retry-After for 429 (seconds or a date) and ignores it elsewhere', () => { + assert.equal(httpError('x', fakeResponse(429, { 'retry-after': '2' })).retryAfterMs, 2000); + const at = new Date(Date.now() + 5000).toUTCString(); + const wait = httpError('x', fakeResponse(429, { 'retry-after': at })).retryAfterMs; + assert.ok(wait > 2000 && wait <= 5000, String(wait)); + assert.equal(httpError('x', fakeResponse(429)).retryAfterMs, undefined); + assert.equal(httpError('x', fakeResponse(400, { 'retry-after': '2' })).retryAfterMs, undefined); +}); + +test('withRetry: repeats once for a transient error, after the configured delay', async (t) => { + const waits = recordSleep(t); + let calls = 0; + const out = await withRetry(async () => { + calls += 1; + if (calls === 1) throw fetchFailed(); + return 'ok'; + }); + assert.equal(out, 'ok'); + assert.equal(calls, 2); + assert.deepEqual(waits, [pollRetry.delayMs]); +}); + +test('withRetry: a second failure is thrown, not retried again', async (t) => { + recordSleep(t); + let calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw httpError('x', fakeResponse(503)); + }), + /x 503/ + ); + assert.equal(calls, 2); +}); + +test('withRetry: 429 waits for Retry-After; a wait that is too long is not waited for', async (t) => { + const waits = recordSleep(t); + let calls = 0; + await withRetry(async () => { + calls += 1; + if (calls === 1) throw httpError('x', fakeResponse(429, { 'retry-after': '4' })); + }); + assert.deepEqual(waits, [4000]); + + waits.length = 0; + calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw httpError('x', fakeResponse(429, { 'retry-after': '3600' })); + }), + /x 429/ + ); + assert.equal(calls, 1); + assert.deepEqual(waits, []); +}); + +test('withRetry: 4xx answers and programming errors are not retried', async (t) => { + const waits = recordSleep(t); + for (const err of [ + httpError('x', fakeResponse(400)), + httpError('x', fakeResponse(401)), + httpError('x', fakeResponse(403)), + new TypeError("Cannot read properties of undefined (reading 'data')"), + ]) { + let calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw err; + }), + (e) => e === err + ); + assert.equal(calls, 1); + } + assert.deepEqual(waits, []); +}); + +test('createPollHealth: warns once after 3 failures in a row, once when it works again', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + const health = createPollHealth('x-alerts'); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 0); + health.failed(fetchFailed('ETIMEDOUT')); + assert.equal(warn.mock.callCount(), 1); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 1, 'silent while it keeps failing'); + health.ok(); + assert.equal(warn.mock.callCount(), 2); + assert.match(warn.mock.calls[1].arguments[1], /works again after 5 failed polls/); + health.ok(); + assert.equal(warn.mock.callCount(), 2, 'a working poll is not logged again'); + assert.equal(error.mock.callCount(), 0); +}); + +test('createPollHealth: a run of failures that ends before the third says nothing', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const health = createPollHealth('x-alerts'); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + health.ok(); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 0); +}); + +// --- ticks ---------------------------------------------------------------------------------- + +function installClient(sent) { + const channel = { + isTextBased: () => true, + permissionsFor: () => ({ has: () => true }), + send: async (payload) => { + sent.push(payload); + return { id: `msg-${sent.length}` }; + }, + }; + const guild = { + id: GUILD, + channels: { cache: new Map([[CH_A, channel]]), fetch: async () => channel }, + members: { me: {} }, + }; + runtime.client = { isReady: () => true, guilds: { cache: new Map([[GUILD, guild]]) } }; + guild.client = runtime.client; +} + +/** + * Run `body` with a fake global fetch. `answer(url, n)` gets the URL and how many requests the + * platform's data endpoints have seen (the token endpoint always works) and returns a Response or throws. + */ +async function withFakeFetch(dataHost, answer, body) { + const realFetch = globalThis.fetch; + const seen = { data: 0 }; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.pathname.includes('token')) return Response.json({ access_token: 't', expires_in: 3600 }); + assert.equal(u.hostname, dataHost); + seen.data += 1; + return answer(u, seen.data); + }; + try { + await body(seen); + } finally { + globalThis.fetch = realFetch; + } +} + +const PLATFORMS = [ + { + name: 'twitch', + mod: twitch, + scope: 'twitch-alerts', + host: 'api.twitch.tv', + save: (value) => + setGuildModule(GUILD, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: value, channelId: CH_A }] }, + }), + liveAnswer: (u, login) => + Response.json({ + data: u.pathname.endsWith('/users') + ? [] + : [ + { + id: `s-${login}`, + user_login: login, + user_name: login, + title: 'hi', + game_name: 'g', + viewer_count: 3, + started_at: '2026-11-16T00:00:00Z', + thumbnail_url: '', + }, + ], + }), + }, + { + name: 'kick', + mod: kick, + scope: 'kick-alerts', + host: 'api.kick.com', + save: (value) => + setGuildModule(GUILD, 'kick-alerts', { + enabled: true, + config: { alerts: [{ slug: value, channelId: CH_A }] }, + }), + liveAnswer: (u, slug) => + Response.json({ + data: [ + { + slug, + stream_title: 'hi', + category: { name: 'g' }, + stream: { + is_live: true, + start_time: '2026-11-16T00:00:00Z', + viewer_count: 2, + url: `https://kick.com/${slug}`, + }, + }, + ], + }), + }, +]; + +for (const p of PLATFORMS) { + const prep = async (t, value) => { + const sent = []; + installClient(sent); + p.mod.pollHealth.reset(); + const waits = recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await p.save(value); + return { sent, waits, warn, error }; + }; + + test(`${p.name} tick: one network blip is retried silently and the stream is announced`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'blipone'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? Promise.reject(fetchFailed('ECONNRESET')) : p.liveAnswer(u, 'blipone')), + async (seen) => { + await p.mod.tick(); + assert.ok(seen.data >= 2, 'the request was repeated'); + } + ); + assert.equal(sent.length, 1, 'announced as usual'); + assert.deepEqual(waits, [pollRetry.delayMs]); + assert.equal(warn.mock.callCount(), 0); + assert.equal(error.mock.callCount(), 0); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + test(`${p.name} tick: a 503 is retried once too`, async (t) => { + const { sent, warn, error } = await prep(t, 'fivehundred'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? new Response('{}', { status: 503 }) : p.liveAnswer(u, 'fivehundred')), + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: 429 waits for Retry-After before the retry`, async (t) => { + const { sent, waits } = await prep(t, 'slowdown'); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? new Response('{}', { status: 429, headers: { 'Retry-After': '3' } }) + : p.liveAnswer(u, 'slowdown'), + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(waits, [3000]); + assert.equal(sent.length, 1); + }); + + test(`${p.name} tick: 3 failed polls in a row are one warning with the cause code, recovery is one more line`, async (t) => { + const { sent, warn, error } = await prep(t, 'flaky'); + let down = true; + await withFakeFetch( + p.host, + (u) => (down ? Promise.reject(fetchFailed('ECONNRESET')) : p.liveAnswer(u, 'flaky')), + async (seen) => { + await p.mod.tick(); + await p.mod.tick(); + assert.equal(seen.data, 4, 'each failed poll tried twice'); + assert.equal(warn.mock.callCount(), 0, 'one or two failed polls are not logged'); + await p.mod.tick(); + assert.equal(warn.mock.callCount(), 1); + const [scope, line] = warn.mock.calls[0].arguments; + assert.equal(scope, p.scope); + assert.match(line, /3 times in a row/); + assert.match(line, /fetch failed \(ECONNRESET\)/); + assert.doesNotMatch(line, /https?:|tsecret|ksecret|Bearer/); + await p.mod.tick(); + await p.mod.tick(); + assert.equal(warn.mock.callCount(), 1, 'no repeat while it stays down'); + assert.equal(sent.length, 0); + + down = false; + await p.mod.tick(); + } + ); + assert.equal(warn.mock.callCount(), 2); + assert.match(warn.mock.calls[1].arguments[1], /works again after 5 failed polls/); + assert.equal(sent.length, 1, 'the recovered poll announces as usual'); + assert.equal(error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a programming error is still an error, and is not retried`, async (t) => { + const { waits, warn, error } = await prep(t, 'buggy'); + await withFakeFetch( + p.host, + () => Promise.reject(new TypeError("Cannot read properties of undefined (reading 'x')")), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 1, 'not retried'); + } + ); + assert.equal(error.mock.callCount(), 1); + assert.equal(error.mock.calls[0].arguments[0], p.scope); + assert.match(error.mock.calls[0].arguments[2], /Cannot read properties/); + assert.equal(warn.mock.callCount(), 0); + assert.deepEqual(waits, []); + assert.equal(p.mod.pollHealth.failures, 0, 'does not count as a network failure'); + }); + + test(`${p.name} tick: a 4xx answer is not retried and still logged as an error`, async (t) => { + const { waits, warn, error } = await prep(t, 'denied'); + await withFakeFetch( + p.host, + () => new Response('{}', { status: 403 }), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 1); + } + ); + assert.equal(error.mock.callCount(), 1); + assert.match(error.mock.calls[0].arguments[2], /403/); + assert.equal(warn.mock.callCount(), 0); + assert.deepEqual(waits, []); + }); +} + +test.after(() => { + runtime.client = null; +}); From 6051ed8b8ef8f2c07e2000ae3d7deef342c68788 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 12:19:02 +0100 Subject: [PATCH 2/3] fix(live-alerts): review follow-ups for the poll error handling Return the tick promise to the scheduler so a slow tick is not started twice. Read Ratelimit-Reset only for a 429. Count only the connection and timeout codes of undici as network errors. Refresh a rejected token and retry once at once. Log an error that is not transient the first time, when it changes and then once an hour, with its stack, and remind hourly during a long network outage. Mark token endpoint errors so a refused token is never read as a rejected value. --- src/modules/kickAlerts.js | 16 +- src/modules/lib/pollErrors.js | 101 +++++++--- src/modules/twitchAlerts.js | 16 +- test/liveAlertsPollErrors.test.js | 303 +++++++++++++++++++++++++++++- 4 files changed, 396 insertions(+), 40 deletions(-) diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index 631c8668..09291a87 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -17,7 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; -import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; +import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -70,6 +70,11 @@ export function normaliseKickConfig(raw = {}) { let token = { value: null, expiresAt: 0 }; +/** Forget the cached app token. For tests. */ +export function _resetToken() { + token = { value: null, expiresAt: 0 }; +} + async function appToken() { if (token.value && Date.now() < token.expiresAt - 60_000) return token.value; const body = new URLSearchParams({ @@ -78,7 +83,7 @@ async function appToken() { client_secret: config.kickClientSecret, }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw httpError('Kick token', res); + if (!res.ok) throw tokenError('Kick token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; @@ -98,7 +103,8 @@ function getChannels(slugs) { }); if (res.status === 401) { token = { value: null, expiresAt: 0 }; // force a refresh next call - throw httpError('Kick', res); + // A rejected token: it was just cleared, so one retry at once gets a fresh one. + throw Object.assign(httpError('Kick', res), { retryNow: true }); } if (!res.ok) throw httpError('Kick /channels', res); return (await res.json()).data ?? []; @@ -226,8 +232,6 @@ export async function tick() { every( 'kick-alerts', POLL_MS, - () => { - tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)); - }, + () => tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)), { firstDelayMs: 30_000 } ); diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js index e008bb1d..78a53b2b 100644 --- a/src/modules/lib/pollErrors.js +++ b/src/modules/lib/pollErrors.js @@ -6,9 +6,12 @@ // 429 (honouring Retry-After). 4xx answers other than 429 are not retried: they mean the // request or a value in it is wrong, and asking again gives the same answer. // - createPollHealth() counts consecutive failed polls. One line (warn) after POLL_WARN_AFTER -// failures in a row, one when polling works again, nothing in between. Errors that are not -// network or HTTP problems (a TypeError in our own code, a 4xx that needs a fix such as bad -// credentials) are still logged as errors every time. +// failures in a row, one when polling works again, and a "still failing" reminder once an hour +// while it lasts. Errors that are not network or HTTP problems (a TypeError in our own code, a +// 4xx that needs a fix such as bad credentials) are logged as errors the first time and when +// the message changes, then once an hour ("still failing after N polls"): every error line +// also feeds /health, the error counter and the error reports, so one bad secret must not +// write 1440 of them a day. // "fetch failed", Node's generic network error, hides the reason in `err.cause`; describeError() // puts its code (ECONNRESET, UND_ERR_CONNECT_TIMEOUT, ...) in the text. import { log } from '../../lib/log.js'; @@ -16,6 +19,9 @@ import { log } from '../../lib/log.js'; /** Failed polls in a row before one warning is logged. */ export const POLL_WARN_AFTER = 3; +/** How often a failure that goes on is logged again. */ +export const STILL_FAILING_MS = 60 * 60_000; + /** Retry settings. An object so a test can shorten the wait; do not change it elsewhere. */ export const pollRetry = { delayMs: 3000, @@ -37,6 +43,15 @@ const NETWORK_CODES = new Set([ 'EHOSTUNREACH', 'EHOSTDOWN', 'ERR_SSL_WRONG_VERSION_NUMBER', + // undici: connection and timeout problems only. The others (UND_ERR_INVALID_ARG, ...) are + // faults in how we call it and must stay errors. + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_CLOSED', + 'UND_ERR_ABORTED', + 'UND_ERR_REQ_RETRY', ]); /** The error, its causes, and the members of an AggregateError (a dual-stack connect failure). */ @@ -70,7 +85,7 @@ export function isNetworkError(err) { return chain(err).some( (e) => isTimeoutName(e) || - (typeof e.code === 'string' && (NETWORK_CODES.has(e.code) || e.code.startsWith('UND_ERR_'))) || + (typeof e.code === 'string' && NETWORK_CODES.has(e.code)) || (e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')) ); } @@ -93,9 +108,12 @@ function retryAfterMs(res) { const date = Date.parse(header); if (Number.isFinite(date)) return Math.max(0, date - Date.now()); } - // Twitch states when its rate limit resets, in epoch seconds. - const reset = Number(res.headers?.get?.('ratelimit-reset')); - if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + // Twitch states when its rate limit resets, in epoch seconds. That says nothing about an + // outage, so it is only read for a 429. + if (res.status === 429) { + const reset = Number(res.headers?.get?.('ratelimit-reset')); + if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + } return null; } @@ -113,13 +131,28 @@ export function httpError(label, res) { return err; } -/** Run `fn`; if it fails with a transient error, wait a moment and run it once more. */ +/** + * The error for a non-OK answer of the token endpoint. Its status is kept only when it is + * transient (`httpStatus` always holds it): a refused token is a fault of the request, never of + * one stored value, so batchIsolation's "rejected value" statuses (400, 404, 422) must not see it. + */ +export function tokenError(label, res) { + const err = httpError(label, res); + err.httpStatus = res.status; + if (!isTransientStatus(res.status)) delete err.status; + return err; +} + +/** + * Run `fn`; if it fails with a transient error, wait a moment and run it once more. An error with + * `retryNow` (a 401 whose token was just cleared) is retried at once. + */ export async function withRetry(fn) { try { return await fn(); } catch (err) { - if (!isTransient(err)) throw err; - const wait = err.retryAfterMs ?? pollRetry.delayMs; + if (!isTransient(err) && !err?.retryNow) throw err; + const wait = err.retryNow ? 0 : (err.retryAfterMs ?? pollRetry.delayMs); if (wait > pollRetry.maxWaitMs) throw err; await pollRetry.sleep(wait); return await fn(); @@ -138,39 +171,63 @@ export function describeError(err) { /** * Counts consecutive failed polls of one platform. * @param {string} scope the log scope, e.g. 'twitch-alerts' + * @param {{ warnAfter?: number, repeatMs?: number, now?: () => number }} [o] */ -export function createPollHealth(scope, { warnAfter = POLL_WARN_AFTER } = {}) { - let failures = 0; +export function createPollHealth( + scope, + { warnAfter = POLL_WARN_AFTER, repeatMs = STILL_FAILING_MS, now = () => Date.now() } = {} +) { + let failures = 0; // transient failures in a row let warned = false; + let warnedAt = 0; + /** The error that is not transient and goes on: its text, polls it failed, when it was last logged. */ + let fault = null; + const reset = () => { + failures = 0; + warned = false; + fault = null; + }; return { - /** Failed polls in a row. */ + /** Transient failed polls in a row. */ get failures() { return failures; }, - /** A poll failed with `err`. Logs now, or only after `warnAfter` in a row, see the file head. */ + /** A poll failed with `err`. Logs now, later or not at all, see the file head. */ failed(err) { if (!isTransient(err)) { - log.error(scope, 'poll failed:', describeError(err)); + const text = describeError(err); + if (!fault || fault.text !== text) { + fault = { text, polls: 1, loggedAt: now() }; + log.error(scope, 'poll failed:', err); // the Error itself: its stack reaches the logs + return; + } + fault.polls += 1; + if (now() - fault.loggedAt >= repeatMs) { + fault.loggedAt = now(); + log.error(scope, `poll still failing after ${fault.polls} polls: ${text}`); + } return; } failures += 1; if (failures === warnAfter) { warned = true; + warnedAt = now(); log.warn( scope, `poll failed ${failures} times in a row (${describeError(err)}); new streams may be announced late until it works again` ); + } else if (warned && now() - warnedAt >= repeatMs) { + warnedAt = now(); + log.warn(scope, `poll still failing after ${failures} polls (${describeError(err)})`); } }, /** A poll worked. */ ok() { - if (warned) log.warn(scope, `polling works again after ${failures} failed polls`); - failures = 0; - warned = false; - }, - reset() { - failures = 0; - warned = false; + if (warned || fault) { + log.warn(scope, `polling works again after ${failures + (fault?.polls ?? 0)} failed polls`); + } + reset(); }, + reset, }; } diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 1b7d12dd..2eb4e5b6 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -17,7 +17,7 @@ import { settleEndedPost } from './lib/liveAlerts.js'; import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; -import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; +import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -66,6 +66,11 @@ export function normaliseTwitchConfig(raw = {}) { let token = { value: null, expiresAt: 0 }; +/** Forget the cached app token. For tests. */ +export function _resetToken() { + token = { value: null, expiresAt: 0 }; +} + async function appToken() { if (token.value && Date.now() < token.expiresAt - 60_000) return token.value; const body = new URLSearchParams({ @@ -74,7 +79,7 @@ async function appToken() { grant_type: 'client_credentials', }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw httpError('Twitch token', res); + if (!res.ok) throw tokenError('Twitch token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; @@ -93,7 +98,8 @@ function helix(path, params) { }); if (res.status === 401) { token = { value: null, expiresAt: 0 }; // force refresh next call - throw httpError('Twitch', res); + // A rejected token: it was just cleared, so one retry at once gets a fresh one. + throw Object.assign(httpError('Twitch', res), { retryNow: true }); } if (!res.ok) throw httpError(`Twitch ${path}`, res); return (await res.json()).data ?? []; @@ -229,8 +235,6 @@ export async function tick() { every( 'twitch-alerts', POLL_MS, - () => { - tick().catch((err) => log.error('twitch-alerts', 'tick failed:', err.message)); - }, + () => tick().catch((err) => log.error('twitch-alerts', 'tick failed:', err.message)), { firstDelayMs: 25_000 } ); diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js index a55a008c..14ae1c86 100644 --- a/test/liveAlertsPollErrors.test.js +++ b/test/liveAlertsPollErrors.test.js @@ -19,8 +19,10 @@ const { httpError, withRetry, createPollHealth, + tokenError, pollRetry, } = pollErrors; +const { startSchedules, stopSchedules } = await import('../src/modules/schedule.js'); const twitch = await import('../src/modules/twitchAlerts.js'); const kick = await import('../src/modules/kickAlerts.js'); @@ -189,6 +191,117 @@ test('createPollHealth: a run of failures that ends before the third says nothin assert.equal(warn.mock.callCount(), 0); }); +test('isNetworkError: only the connection and timeout codes of undici count', () => { + const coded = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error('x'), { code }) }); + for (const code of [ + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + ]) { + assert.equal(isNetworkError(coded(code)), true, code); + } + for (const code of ['UND_ERR_INVALID_ARG', 'UND_ERR_NOT_SUPPORTED', 'UND_ERR_INVALID_RETURN_VALUE']) { + const err = Object.assign(new Error('bad call'), { code }); + assert.equal(isNetworkError(err), false, code); + assert.equal(isTransient(err), false, code); + } +}); + +test('httpError: Ratelimit-Reset is only the wait for a 429, never for a 503', () => { + const reset = String(Math.floor(Date.now() / 1000) + 8); + const wait = httpError('x', fakeResponse(429, { 'ratelimit-reset': reset })).retryAfterMs; + assert.ok(wait > 4000 && wait <= 8000, String(wait)); + assert.equal(httpError('x', fakeResponse(503, { 'ratelimit-reset': reset })).retryAfterMs, undefined); + assert.equal(httpError('x', fakeResponse(503, { 'retry-after': '2' })).retryAfterMs, 2000); + const past = new Date(Date.now() - 60_000).toUTCString(); + assert.equal(httpError('x', fakeResponse(503, { 'retry-after': past })).retryAfterMs, 0); +}); + +test('tokenError: a refused token never carries a status that batch isolation reads as a value fault', () => { + for (const status of [400, 401, 403, 404, 422]) { + const err = tokenError('Twitch token', fakeResponse(status)); + assert.equal(err.status, undefined, String(status)); + assert.equal(err.httpStatus, status); + assert.equal(err.message, `Twitch token ${status}`); + assert.equal(isTransient(err), false); + } + for (const status of [429, 500, 503]) { + const err = tokenError('Twitch token', fakeResponse(status)); + assert.equal(err.status, status); + assert.equal(isTransient(err), true); + } +}); + +test('withRetry: an error marked retryNow is retried at once, but only once', async (t) => { + const waits = recordSleep(t); + let calls = 0; + await withRetry(async () => { + calls += 1; + if (calls === 1) throw Object.assign(httpError('x', fakeResponse(401)), { retryNow: true }); + }); + assert.equal(calls, 2); + assert.deepEqual(waits, [0]); + calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw Object.assign(httpError('x', fakeResponse(401)), { retryNow: true }); + }), + /x 401/ + ); + assert.equal(calls, 2); +}); + +test('createPollHealth: an error that is not transient is logged once, again when it changes, then hourly', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + let now = 1_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + const denied = () => httpError('Twitch /streams', fakeResponse(403)); + const bug = new TypeError("Cannot read properties of undefined (reading 'x')"); + health.failed(bug); + assert.equal(error.mock.callCount(), 1); + assert.equal(error.mock.calls[0].arguments[2], bug, 'the Error itself, so its stack is logged'); + for (let i = 0; i < 100; i += 1) { + now += 60_000; + health.failed(bug); + } + assert.equal(error.mock.callCount(), 2, 'silent for the first hour, then one reminder'); + assert.match(error.mock.calls[1].arguments[1], /still failing after \d+ polls: Cannot read properties/); + health.failed(denied()); // a different message: logged at once + assert.equal(error.mock.callCount(), 3); + now += 61 * 60_000; + health.failed(denied()); + health.failed(denied()); + assert.equal(error.mock.callCount(), 4); + assert.match(error.mock.calls[3].arguments[1], /still failing after 2 polls: Twitch \/streams 403/); + assert.equal(health.failures, 0, 'not a network failure'); + health.ok(); + assert.equal(warn.mock.callCount(), 1); + assert.match(warn.mock.calls[0].arguments[1], /works again after 3 failed polls/); + health.ok(); + assert.equal(warn.mock.callCount(), 1); +}); + +test('createPollHealth: a long network outage gets a reminder every hour', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + let now = 5_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + for (let i = 0; i < 130; i += 1) { + health.failed(fetchFailed('ENOTFOUND')); + now += 60_000; + } + // 130 minutes: the warning at the 3rd failure, then one reminder per hour + assert.equal(warn.mock.callCount(), 3); + assert.match(warn.mock.calls[0].arguments[1], /3 times in a row/); + assert.match( + warn.mock.calls[1].arguments[1], + /still failing after \d+ polls \(fetch failed \(ENOTFOUND\)\)/ + ); + assert.match(warn.mock.calls[2].arguments[1], /still failing after \d+ polls/); +}); + // --- ticks ---------------------------------------------------------------------------------- function installClient(sent) { @@ -209,16 +322,22 @@ function installClient(sent) { guild.client = runtime.client; } +const tokenOk = () => Response.json({ access_token: 't', expires_in: 3600 }); + /** * Run `body` with a fake global fetch. `answer(url, n)` gets the URL and how many requests the - * platform's data endpoints have seen (the token endpoint always works) and returns a Response or throws. + * platform's data endpoints have seen and returns a Response or throws. `token(n)` answers the n-th + * request of the token endpoint (default: it works). */ -async function withFakeFetch(dataHost, answer, body) { +async function withFakeFetch(dataHost, answer, body, { token } = {}) { const realFetch = globalThis.fetch; - const seen = { data: 0 }; + const seen = { data: 0, token: 0 }; globalThis.fetch = async (url) => { const u = new URL(String(url)); - if (u.pathname.includes('token')) return Response.json({ access_token: 't', expires_in: 3600 }); + if (u.pathname.includes('token')) { + seen.token += 1; + return token ? token(seen.token) : tokenOk(); + } assert.equal(u.hostname, dataHost); seen.data += 1; return answer(u, seen.data); @@ -293,6 +412,7 @@ for (const p of PLATFORMS) { const sent = []; installClient(sent); p.mod.pollHealth.reset(); + p.mod._resetToken(); const waits = recordSleep(t); const warn = t.mock.method(log, 'warn', () => {}); const error = t.mock.method(log, 'error', () => {}); @@ -391,7 +511,8 @@ for (const p of PLATFORMS) { ); assert.equal(error.mock.callCount(), 1); assert.equal(error.mock.calls[0].arguments[0], p.scope); - assert.match(error.mock.calls[0].arguments[2], /Cannot read properties/); + assert.match(error.mock.calls[0].arguments[2].message, /Cannot read properties/); + assert.ok(error.mock.calls[0].arguments[2].stack, 'the Error itself is logged, with its stack'); assert.equal(warn.mock.callCount(), 0); assert.deepEqual(waits, []); assert.equal(p.mod.pollHealth.failures, 0, 'does not count as a network failure'); @@ -408,12 +529,182 @@ for (const p of PLATFORMS) { } ); assert.equal(error.mock.callCount(), 1); - assert.match(error.mock.calls[0].arguments[2], /403/); + assert.match(error.mock.calls[0].arguments[2].message, /403/); assert.equal(warn.mock.callCount(), 0); assert.deepEqual(waits, []); }); + + const tokenBlips = [ + ['a 503', () => new Response('{}', { status: 503 })], + ['a network blip', () => Promise.reject(fetchFailed('ECONNRESET'))], + ]; + for (const [i, [what, first]] of tokenBlips.entries()) { + test(`${p.name} tick: the token endpoint answering with ${what} is retried silently`, async (t) => { + const { sent, waits, warn, error } = await prep(t, `tokenblip${i}`); + await withFakeFetch( + p.host, + (u) => p.liveAnswer(u, `tokenblip${i}`), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 2, 'the token request was repeated'); + }, + { token: (n) => (n === 1 ? first() : tokenOk()) } + ); + assert.equal(sent.length, 1); + assert.deepEqual(waits, [pollRetry.delayMs]); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + } + + test(`${p.name} tick: a token endpoint that refuses with 400 is one error, with no search and no data request`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'badclient'); + await withFakeFetch( + p.host, + (u) => p.liveAnswer(u, 'badclient'), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 1, 'one token request, not one per isolated batch'); + assert.equal(seen.data, 0); + }, + { token: () => new Response('{"error":"invalid client"}', { status: 400 }) } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 1); + assert.match(error.mock.calls[0].arguments[2].message, /token 400/); + assert.equal(warn.mock.callCount(), 0, 'no "every request was refused" warning'); + assert.deepEqual(waits, []); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + const retryAfters = [ + ['seconds', '2', 2000], + ['a date that has passed', new Date(Date.now() - 60_000).toUTCString(), 0], + ]; + for (const [i, [what, header, expected]] of retryAfters.entries()) { + test(`${p.name} tick: a 503 honours Retry-After given as ${what}`, async (t) => { + const { sent, waits, warn, error } = await prep(t, `retryafter${i}`); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? new Response('{}', { status: 503, headers: { 'Retry-After': header } }) + : p.liveAnswer(u, `retryafter${i}`), + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(waits, [expected]); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + } + + test(`${p.name} tick: a request timeout is retried silently`, async (t) => { + const { sent, warn, error } = await prep(t, 'timedout'); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? Promise.reject(new DOMException('The operation was aborted due to timeout', 'TimeoutError')) + : p.liveAnswer(u, 'timedout'), + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a rejected token (401) is refreshed and the request retried at once, silently`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'rotated'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? new Response('{}', { status: 401 }) : p.liveAnswer(u, 'rotated')), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 2, 'a fresh token for the retry'); + } + ); + assert.deepEqual(waits, [0]); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a 401 that stays is an error`, async (t) => { + const { sent, error } = await prep(t, 'norights'); + await withFakeFetch( + p.host, + () => new Response('{}', { status: 401 }), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 2, 'tried once more, not more'); + } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 1); + }); + + test(`${p.name} tick: bad credentials are one error line for many polls, then one recovery line`, async (t) => { + const { sent, warn, error } = await prep(t, 'badsecret'); + let denied = true; + await withFakeFetch( + p.host, + (u) => (denied ? new Response('{}', { status: 403 }) : p.liveAnswer(u, 'badsecret')), + async () => { + for (let i = 0; i < 5; i += 1) await p.mod.tick(); + assert.equal(error.mock.callCount(), 1, 'logged once, not on every poll'); + denied = false; + await p.mod.tick(); + } + ); + assert.equal(warn.mock.callCount(), 1); + assert.match(warn.mock.calls[0].arguments[1], /works again after 5 failed polls/); + assert.equal(sent.length, 1); + }); } +test('the poll callbacks hand their tick to the scheduler: a tick that is still running is not started twice', async (t) => { + const sent = []; + installClient(sent); + recordSleep(t); + t.mock.method(log, 'warn', () => {}); + t.mock.method(log, 'error', () => {}); + for (const p of PLATFORMS) { + p.mod.pollHealth.reset(); + p.mod._resetToken(); + await p.save('slowtick'); + } + const hits = {}; + const release = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (url) => { + const u = new URL(String(url)); + if (u.pathname.includes('token')) return Promise.resolve(tokenOk()); + hits[u.hostname] = (hits[u.hostname] ?? 0) + 1; + const p = PLATFORMS.find((x) => x.host === u.hostname); + return new Promise((resolve) => release.push(() => resolve(p.liveAnswer(u, 'slowtick')))); + }; + const flush = async () => { + for (let i = 0; i < 30; i += 1) await new Promise((resolve) => setImmediate(resolve)); + }; + t.mock.timers.enable({ apis: ['setTimeout', 'setInterval'] }); + try { + startSchedules(); + t.mock.timers.tick(30_000); // the first ticks (Twitch 25 s, Kick 30 s), now waiting on the network + await flush(); + t.mock.timers.tick(60_000); // the next interval comes while they are still waiting + await flush(); + assert.deepEqual(hits, { 'api.twitch.tv': 1, 'api.kick.com': 1 }, 'no second tick on top of the first'); + for (let i = 0; i < 10 && release.length > 0; i += 1) { + for (const go of release.splice(0)) go(); + await flush(); + } + await stopSchedules(); + } finally { + globalThis.fetch = realFetch; + } +}); + test.after(() => { runtime.client = null; }); From e1292fa841393dbf93d5cd57bf34b81f902e30b0 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 12:41:15 +0100 Subject: [PATCH 3/3] fix(live-alerts): classify fetch failures by their cause code and key the error throttle by kind A TypeError('fetch failed') with a cause is now a network error only when the cause code is a connection or timeout code; UND_ERR_INVALID_ARG and similar stay errors. The hourly throttle for errors that are not transient keys on name, status and the message without numbers, so alternating errors or a changing value in the message are not logged on every poll. --- src/modules/lib/pollErrors.js | 43 +++++++++++++++++++++---------- test/liveAlertsPollErrors.test.js | 40 +++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js index 78a53b2b..e45633f3 100644 --- a/src/modules/lib/pollErrors.js +++ b/src/modules/lib/pollErrors.js @@ -82,11 +82,15 @@ export function causeCode(err) { export function isNetworkError(err) { if (!err || typeof err !== 'object') return false; if (typeof err.status === 'number') return false; // we got an answer - return chain(err).some( - (e) => - isTimeoutName(e) || - (typeof e.code === 'string' && NETWORK_CODES.has(e.code)) || - (e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')) + const items = chain(err); + if (items.some(isTimeoutName)) return true; + // undici wraps every dispatcher error in TypeError('fetch failed', { cause }): when a cause + // carries a code, the code decides (UND_ERR_INVALID_ARG is our fault, ECONNRESET is the network). + const codes = items.map((e) => e.code).filter((c) => typeof c === 'string' && c); + if (codes.length > 0) return codes.some((c) => NETWORK_CODES.has(c)); + // No code anywhere: the generic wrapper alone counts as a network failure. + return items.some( + (e) => e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated') ); } @@ -180,12 +184,21 @@ export function createPollHealth( let failures = 0; // transient failures in a row let warned = false; let warnedAt = 0; - /** The error that is not transient and goes on: its text, polls it failed, when it was last logged. */ - let fault = null; + /** + * The errors that are not transient and go on, by kind (name, status, message without numbers), + * so errors that alternate, or whose message holds a changing value, are not logged every poll. + * A fixed number of kinds is kept; the oldest is dropped. + * @type {Map} + */ + const faults = new Map(); + const MAX_KINDS = 10; + const kindOf = (err) => + [err?.name, err?.httpStatus ?? err?.status ?? '', describeError(err).replace(/\d+/g, '#')].join('|'); + const faultPolls = () => [...faults.values()].reduce((sum, f) => sum + f.polls, 0); const reset = () => { failures = 0; warned = false; - fault = null; + faults.clear(); }; return { /** Transient failed polls in a row. */ @@ -195,16 +208,18 @@ export function createPollHealth( /** A poll failed with `err`. Logs now, later or not at all, see the file head. */ failed(err) { if (!isTransient(err)) { - const text = describeError(err); - if (!fault || fault.text !== text) { - fault = { text, polls: 1, loggedAt: now() }; + const kind = kindOf(err); + const fault = faults.get(kind); + if (!fault) { + if (faults.size >= MAX_KINDS) faults.delete(faults.keys().next().value); + faults.set(kind, { polls: 1, loggedAt: now() }); log.error(scope, 'poll failed:', err); // the Error itself: its stack reaches the logs return; } fault.polls += 1; if (now() - fault.loggedAt >= repeatMs) { fault.loggedAt = now(); - log.error(scope, `poll still failing after ${fault.polls} polls: ${text}`); + log.error(scope, `poll still failing after ${fault.polls} polls: ${describeError(err)}`); } return; } @@ -223,8 +238,8 @@ export function createPollHealth( }, /** A poll worked. */ ok() { - if (warned || fault) { - log.warn(scope, `polling works again after ${failures + (fault?.polls ?? 0)} failed polls`); + if (warned || faults.size > 0) { + log.warn(scope, `polling works again after ${failures + faultPolls()} failed polls`); } reset(); }, diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js index 14ae1c86..ef12236a 100644 --- a/test/liveAlertsPollErrors.test.js +++ b/test/liveAlertsPollErrors.test.js @@ -208,6 +208,22 @@ test('isNetworkError: only the connection and timeout codes of undici count', () } }); +test('isNetworkError: a fetch failure with a cause is classified by the cause code', () => { + const wrapped = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error('x'), { code }) }); + for (const code of ['ECONNRESET', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'ENOTFOUND']) { + assert.equal(isNetworkError(wrapped(code)), true, code); + assert.equal(isTransient(wrapped(code)), true, code); + } + for (const code of ['UND_ERR_INVALID_ARG', 'UND_ERR_NOT_SUPPORTED', 'UND_ERR_INVALID_RETURN_VALUE']) { + assert.equal(isNetworkError(wrapped(code)), false, code); + assert.equal(isTransient(wrapped(code)), false, code); + } + // the wrapper alone, or with a cause that has no code, still counts as a network failure + assert.equal(isNetworkError(new TypeError('fetch failed')), true); + assert.equal(isNetworkError(new TypeError('fetch failed', { cause: new Error('no code') })), true); + assert.equal(isNetworkError(new TypeError('Cannot read properties of undefined')), false); +}); + test('httpError: Ratelimit-Reset is only the wait for a 429, never for a 503', () => { const reset = String(Math.floor(Date.now() / 1000) + 8); const wait = httpError('x', fakeResponse(429, { 'ratelimit-reset': reset })).retryAfterMs; @@ -279,11 +295,33 @@ test('createPollHealth: an error that is not transient is logged once, again whe assert.equal(health.failures, 0, 'not a network failure'); health.ok(); assert.equal(warn.mock.callCount(), 1); - assert.match(warn.mock.calls[0].arguments[1], /works again after 3 failed polls/); + assert.match(warn.mock.calls[0].arguments[1], /works again after 104 failed polls/); health.ok(); assert.equal(warn.mock.callCount(), 1); }); +test('createPollHealth: alternating errors, or a message with a changing number, are not logged on every poll', (t) => { + const error = t.mock.method(log, 'error', () => {}); + let now = 9_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + const a = () => httpError('Twitch /streams', fakeResponse(403)); + const b = () => httpError('Twitch token', fakeResponse(401)); + for (let i = 0; i < 20; i += 1) { + health.failed(a()); + health.failed(b()); + now += 60_000; + } + assert.equal(error.mock.callCount(), 2, 'one line for each of the two kinds'); + for (let i = 0; i < 30; i += 1) { + health.failed(new RangeError(`index ${i} is out of range 0..${100 + i}`)); + now += 60_000; + } + assert.equal(error.mock.callCount(), 3, 'the number in the message does not make a new kind'); + // a different status is a different kind + health.failed(httpError('Twitch /streams', fakeResponse(404))); + assert.equal(error.mock.callCount(), 4); +}); + test('createPollHealth: a long network outage gets a reminder every hour', (t) => { const warn = t.mock.method(log, 'warn', () => {}); let now = 5_000_000;