From 64221bf7c79d6128d0a2c9c32b13ce30ed65991e Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 11:46:39 +0100 Subject: [PATCH 1/3] fix(live-alerts): retry a failed Twitch/Kick poll once and log a run of failures once A single network blip ("fetch failed") no longer logs an error. A failed request is retried once after a short delay for network errors, 5xx and 429 (honouring Retry-After); the retry sits inside the request, so the batch isolation only sees an error after the retry failed too. Failed polls are counted per platform: one warning after 3 in a row, naming the cause code (e.g. ECONNRESET), and one line when polling works again. Programming errors and 4xx answers are still logged as errors. --- src/modules/kickAlerts.js | 41 ++- src/modules/lib/pollErrors.js | 176 ++++++++++ src/modules/twitchAlerts.js | 48 ++- test/liveAlertsPollErrors.test.js | 530 ++++++++++++++++++++++++++++++ 4 files changed, 764 insertions(+), 31 deletions(-) create mode 100644 src/modules/lib/pollErrors.js create mode 100644 test/liveAlertsPollErrors.test.js diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index f5a3a6a8..558d0378 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -18,6 +18,7 @@ import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveVal import { log } from '../lib/log.js'; import { every } from './schedule.js'; import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; +import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -78,28 +79,34 @@ async function appToken() { client_secret: config.kickClientSecret, }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`Kick token ${res.status}`); + if (!res.ok) throw httpError('Kick token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; } -async function getChannels(slugs) { +/** + * One channels request. A network error, 5xx or 429 is retried once (see lib/pollErrors.js); the + * retry sits inside the request, so fetchIsolated only sees an error once the retry failed too. + */ +function getChannels(slugs) { const url = new URL(`${API}/channels`); for (const s of slugs) url.searchParams.append('slug', s); - const res = await fetch(url, { - headers: { - Authorization: `Bearer ${await appToken()}`, - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(10_000), + return withRetry(async () => { + const res = await fetch(url, { + headers: { + Authorization: `Bearer ${await appToken()}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }); + if (res.status === 401) { + token = { value: null, expiresAt: 0 }; // force a refresh next call + throw httpError('Kick', res); + } + if (!res.ok) throw httpError('Kick /channels', res); + return (await res.json()).data ?? []; }); - if (res.status === 401) { - token = { value: null, expiresAt: 0 }; // force a refresh next call - throw new Error('Kick 401'); - } - if (!res.ok) throw Object.assign(new Error(`Kick /channels ${res.status}`), { status: res.status }); - return (await res.json()).data ?? []; } /** @@ -183,6 +190,9 @@ export function buildPayload(channel, alert) { // --- poll loop ------------------------------------------------------- +/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */ +export const pollHealth = createPollHealth('kick-alerts'); + /** One poll; exported for tests. */ export async function tick() { if (!allClientsReady() || !config.kickEnabled) return; @@ -230,9 +240,10 @@ export async function tick() { }, }); } catch (err) { - log.error('kick-alerts', 'poll failed:', err.message); + pollHealth.failed(err); return; } + pollHealth.ok(); for (const { guildId, alert } of jobs) { // Not looked up this poll (rejected by Kick, waiting out a backoff, or over the request diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js new file mode 100644 index 00000000..e008bb1d --- /dev/null +++ b/src/modules/lib/pollErrors.js @@ -0,0 +1,176 @@ +// Transient errors in the shared live-alert polls (Twitch and Kick). +// +// A poll asks a platform for every saved streamer at once, once a minute. A network blip (a reset +// connection, a DNS hiccup, a 502) should not page the operator, so: +// - withRetry() repeats a failed request once after a short delay, for network errors, 5xx and +// 429 (honouring Retry-After). 4xx answers other than 429 are not retried: they mean the +// request or a value in it is wrong, and asking again gives the same answer. +// - createPollHealth() counts consecutive failed polls. One line (warn) after POLL_WARN_AFTER +// failures in a row, one when polling works again, nothing in between. Errors that are not +// network or HTTP problems (a TypeError in our own code, a 4xx that needs a fix such as bad +// credentials) are still logged as errors every time. +// "fetch failed", Node's generic network error, hides the reason in `err.cause`; describeError() +// puts its code (ECONNRESET, UND_ERR_CONNECT_TIMEOUT, ...) in the text. +import { log } from '../../lib/log.js'; + +/** Failed polls in a row before one warning is logged. */ +export const POLL_WARN_AFTER = 3; + +/** Retry settings. An object so a test can shorten the wait; do not change it elsewhere. */ +export const pollRetry = { + delayMs: 3000, + /** A longer Retry-After than this is not waited for: the poll fails and the next one tries again. */ + maxWaitMs: 15_000, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +}; + +const NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', + 'ENETUNREACH', + 'ENETDOWN', + 'EHOSTUNREACH', + 'EHOSTDOWN', + 'ERR_SSL_WRONG_VERSION_NUMBER', +]); + +/** The error, its causes, and the members of an AggregateError (a dual-stack connect failure). */ +function chain(err) { + const out = []; + const queue = [err]; + while (queue.length > 0 && out.length < 8) { + const e = queue.shift(); + if (!e || typeof e !== 'object' || out.includes(e)) continue; + out.push(e); + queue.push(e.cause); + if (Array.isArray(e.errors)) queue.push(...e.errors); + } + return out; +} + +const isTimeoutName = (e) => e?.name === 'TimeoutError' || e?.name === 'AbortError'; + +/** The first string error code in the chain (ECONNRESET, UND_ERR_SOCKET, ...), or the timeout name. */ +export function causeCode(err) { + const items = chain(err); + for (const e of items) if (typeof e.code === 'string' && e.code) return e.code; + const named = items.find(isTimeoutName); + return named ? named.name : null; +} + +/** A failed request that never got an HTTP answer: connection, DNS, TLS, timeout, broken body. */ +export function isNetworkError(err) { + if (!err || typeof err !== 'object') return false; + if (typeof err.status === 'number') return false; // we got an answer + return chain(err).some( + (e) => + isTimeoutName(e) || + (typeof e.code === 'string' && (NETWORK_CODES.has(e.code) || e.code.startsWith('UND_ERR_'))) || + (e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')) + ); +} + +/** 5xx and 429: the platform is busy or down, try again later. Other statuses are not transient. */ +export function isTransientStatus(status) { + return typeof status === 'number' && (status >= 500 || status === 429); +} + +/** Retry once, wait a moment, and count on the next poll: network errors, 5xx and 429. */ +export function isTransient(err) { + return isNetworkError(err) || isTransientStatus(err?.status); +} + +function retryAfterMs(res) { + const header = res.headers?.get?.('retry-after'); + if (header) { + const seconds = Number(header); + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); + const date = Date.parse(header); + if (Number.isFinite(date)) return Math.max(0, date - Date.now()); + } + // Twitch states when its rate limit resets, in epoch seconds. + const reset = Number(res.headers?.get?.('ratelimit-reset')); + if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + return null; +} + +/** + * The error for a non-OK answer: `${label} ${status}`, with `status`, and `retryAfterMs` when the + * platform said how long to wait (429 and 503 only). The message holds no URL, so no credentials. + */ +export function httpError(label, res) { + const err = new Error(`${label} ${res.status}`); + err.status = res.status; + if (res.status === 429 || res.status === 503) { + const wait = retryAfterMs(res); + if (wait !== null) err.retryAfterMs = wait; + } + return err; +} + +/** Run `fn`; if it fails with a transient error, wait a moment and run it once more. */ +export async function withRetry(fn) { + try { + return await fn(); + } catch (err) { + if (!isTransient(err)) throw err; + const wait = err.retryAfterMs ?? pollRetry.delayMs; + if (wait > pollRetry.maxWaitMs) throw err; + await pollRetry.sleep(wait); + return await fn(); + } +} + +/** Short text for a log line: the message, plus the cause code when the message hides it. */ +export function describeError(err) { + const message = String(err?.message ?? err) + .replace(/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi, '') // never a URL, it may carry credentials + .slice(0, 200); + const code = causeCode(err); + return code && !message.includes(code) ? `${message} (${code})` : message; +} + +/** + * Counts consecutive failed polls of one platform. + * @param {string} scope the log scope, e.g. 'twitch-alerts' + */ +export function createPollHealth(scope, { warnAfter = POLL_WARN_AFTER } = {}) { + let failures = 0; + let warned = false; + return { + /** Failed polls in a row. */ + get failures() { + return failures; + }, + /** A poll failed with `err`. Logs now, or only after `warnAfter` in a row, see the file head. */ + failed(err) { + if (!isTransient(err)) { + log.error(scope, 'poll failed:', describeError(err)); + return; + } + failures += 1; + if (failures === warnAfter) { + warned = true; + log.warn( + scope, + `poll failed ${failures} times in a row (${describeError(err)}); new streams may be announced late until it works again` + ); + } + }, + /** A poll worked. */ + ok() { + if (warned) log.warn(scope, `polling works again after ${failures} failed polls`); + failures = 0; + warned = false; + }, + reset() { + failures = 0; + warned = false; + }, + }; +} diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 7a1056a2..816d9da2 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -18,6 +18,7 @@ import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; +import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -74,27 +75,35 @@ async function appToken() { grant_type: 'client_credentials', }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`Twitch token ${res.status}`); + if (!res.ok) throw httpError('Twitch token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; } -async function helix(path, params) { +/** + * One Helix request. A network error, 5xx or 429 is retried once (see lib/pollErrors.js); the + * retry sits inside the request, so fetchIsolated only sees an error once the retry failed too. + * `retry: false` for requests that only decorate an announcement. + */ +function helix(path, params, { retry = true } = {}) { const url = new URL(HELIX + path); for (const [k, vals] of Object.entries(params)) { for (const v of [].concat(vals)) url.searchParams.append(k, v); } - const res = await fetch(url, { - headers: { 'Client-Id': config.twitchClientId, Authorization: `Bearer ${await appToken()}` }, - signal: AbortSignal.timeout(10_000), - }); - if (res.status === 401) { - token = { value: null, expiresAt: 0 }; // force refresh next call - throw new Error('Twitch 401'); - } - if (!res.ok) throw Object.assign(new Error(`Twitch ${path} ${res.status}`), { status: res.status }); - return (await res.json()).data ?? []; + const once = async () => { + const res = await fetch(url, { + headers: { 'Client-Id': config.twitchClientId, Authorization: `Bearer ${await appToken()}` }, + signal: AbortSignal.timeout(10_000), + }); + if (res.status === 401) { + token = { value: null, expiresAt: 0 }; // force refresh next call + throw httpError('Twitch', res); + } + if (!res.ok) throw httpError(`Twitch ${path}`, res); + return (await res.json()).data ?? []; + }; + return retry ? withRetry(once) : once(); } /** @@ -128,9 +137,12 @@ export async function fetchLiveStreams(logins, { report, groupOf } = {}) { export async function fetchUsers(logins) { const out = new Map(); try { - const { results } = await fetchIsolated(logins, 100, (batch) => helix('/users', { login: batch }), { - quarantine: createQuarantine(), - }); + const { results } = await fetchIsolated( + logins, + 100, + (batch) => helix('/users', { login: batch }, { retry: false }), + { quarantine: createQuarantine() } + ); for (const u of results) out.set(u.login.toLowerCase(), u); } catch (err) { log.debug('twitch-alerts', 'user lookup failed:', err.message); @@ -185,6 +197,9 @@ export function buildPayload(stream, user, alert) { // --- poll loop ------------------------------------------------------- +/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */ +export const pollHealth = createPollHealth('twitch-alerts'); + /** One poll; exported for tests. */ export async function tick() { if (!allClientsReady() || !config.twitchEnabled) return; @@ -234,9 +249,10 @@ export async function tick() { }); users = await fetchUsers([...streams.keys()]); } catch (err) { - log.error('twitch-alerts', 'poll failed:', err.message); + pollHealth.failed(err); return; } + pollHealth.ok(); for (const { guildId, alert } of jobs) { // Not looked up this poll (rejected by Twitch, waiting out a backoff, or over the request diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js new file mode 100644 index 00000000..41f05cc4 --- /dev/null +++ b/test/liveAlertsPollErrors.test.js @@ -0,0 +1,530 @@ +// A network blip in the Twitch or Kick poll must not reach the operator: it is retried once, and a +// run of failed polls is one warning, with the cause code in it. +process.env.TWITCH_CLIENT_ID = 'tid'; +process.env.TWITCH_CLIENT_SECRET = 'tsecret'; +process.env.KICK_CLIENT_ID = 'kid'; +process.env.KICK_CLIENT_SECRET = 'ksecret'; +await import('./helpers/tmpDb.js'); +const { default: test } = await import('node:test'); +const { default: assert } = await import('node:assert/strict'); +const { runtime } = await import('../src/runtime.js'); +const { log } = await import('../src/lib/log.js'); +const { setGuildModule } = await import('../src/db/modules.js'); +const pollErrors = await import('../src/modules/lib/pollErrors.js'); +const { + causeCode, + describeError, + isNetworkError, + isTransient, + httpError, + withRetry, + createPollHealth, + pollRetry, +} = pollErrors; +const twitch = await import('../src/modules/twitchAlerts.js'); +const kick = await import('../src/modules/kickAlerts.js'); + +const CH_A = '100000000000000001'; +const GUILD = '900000000000000601'; + +/** What undici throws for a dropped connection: the reason is in `cause`. */ +const fetchFailed = (code = 'ECONNRESET') => + new TypeError('fetch failed', { cause: Object.assign(new Error(`read ${code}`), { code }) }); +const fakeResponse = (status, headers = {}) => ({ + status, + ok: status >= 200 && status < 300, + headers: new Headers(headers), +}); + +function recordSleep(t) { + const waits = []; + t.mock.method(pollRetry, 'sleep', async (ms) => void waits.push(ms)); + return waits; +} + +// --- helper --------------------------------------------------------------------------------- + +test('describeError: puts the cause code of "fetch failed" in the text', () => { + assert.equal(describeError(fetchFailed('ECONNRESET')), 'fetch failed (ECONNRESET)'); + assert.equal( + describeError(fetchFailed('UND_ERR_CONNECT_TIMEOUT')), + 'fetch failed (UND_ERR_CONNECT_TIMEOUT)' + ); + assert.equal(describeError(fetchFailed('ENOTFOUND')), 'fetch failed (ENOTFOUND)'); + // dual-stack connect failures hide the code one level further down, in an AggregateError + const aggregate = new TypeError('fetch failed', { + cause: new AggregateError([Object.assign(new Error('x'), { code: 'ETIMEDOUT' })]), + }); + assert.equal(describeError(aggregate), 'fetch failed (ETIMEDOUT)'); + assert.equal(causeCode(new Error('plain')), null); +}); + +test('describeError: a timeout names itself, and no URL or credential gets into the text', () => { + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + assert.match(describeError(timeout), /\(TimeoutError\)$/); + const text = describeError(new Error('request to https://user:hunter2@api.example.com/x?token=abc failed')); + assert.doesNotMatch(text, /hunter2|token=abc|https:/); + assert.equal(describeError(httpError('Twitch /streams', fakeResponse(503))), 'Twitch /streams 503'); +}); + +test('isTransient: network errors, 5xx and 429 only', () => { + assert.equal(isNetworkError(fetchFailed()), true); + assert.equal(isNetworkError(new DOMException('x', 'TimeoutError')), true); + assert.equal(isNetworkError(new TypeError("Cannot read properties of undefined (reading 'data')")), false); + assert.equal(isNetworkError(new SyntaxError('Unexpected token < in JSON')), false); + for (const status of [500, 502, 503, 504, 429]) { + assert.equal(isTransient(httpError('x', fakeResponse(status))), true, String(status)); + } + for (const status of [400, 401, 403, 404, 422]) { + assert.equal(isTransient(httpError('x', fakeResponse(status))), false, String(status)); + } +}); + +test('httpError: reads Retry-After for 429 (seconds or a date) and ignores it elsewhere', () => { + assert.equal(httpError('x', fakeResponse(429, { 'retry-after': '2' })).retryAfterMs, 2000); + const at = new Date(Date.now() + 5000).toUTCString(); + const wait = httpError('x', fakeResponse(429, { 'retry-after': at })).retryAfterMs; + assert.ok(wait > 2000 && wait <= 5000, String(wait)); + assert.equal(httpError('x', fakeResponse(429)).retryAfterMs, undefined); + assert.equal(httpError('x', fakeResponse(400, { 'retry-after': '2' })).retryAfterMs, undefined); +}); + +test('withRetry: repeats once for a transient error, after the configured delay', async (t) => { + const waits = recordSleep(t); + let calls = 0; + const out = await withRetry(async () => { + calls += 1; + if (calls === 1) throw fetchFailed(); + return 'ok'; + }); + assert.equal(out, 'ok'); + assert.equal(calls, 2); + assert.deepEqual(waits, [pollRetry.delayMs]); +}); + +test('withRetry: a second failure is thrown, not retried again', async (t) => { + recordSleep(t); + let calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw httpError('x', fakeResponse(503)); + }), + /x 503/ + ); + assert.equal(calls, 2); +}); + +test('withRetry: 429 waits for Retry-After; a wait that is too long is not waited for', async (t) => { + const waits = recordSleep(t); + let calls = 0; + await withRetry(async () => { + calls += 1; + if (calls === 1) throw httpError('x', fakeResponse(429, { 'retry-after': '4' })); + }); + assert.deepEqual(waits, [4000]); + + waits.length = 0; + calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw httpError('x', fakeResponse(429, { 'retry-after': '3600' })); + }), + /x 429/ + ); + assert.equal(calls, 1); + assert.deepEqual(waits, []); +}); + +test('withRetry: 4xx answers and programming errors are not retried', async (t) => { + const waits = recordSleep(t); + for (const err of [ + httpError('x', fakeResponse(400)), + httpError('x', fakeResponse(401)), + httpError('x', fakeResponse(403)), + new TypeError("Cannot read properties of undefined (reading 'data')"), + ]) { + let calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw err; + }), + (e) => e === err + ); + assert.equal(calls, 1); + } + assert.deepEqual(waits, []); +}); + +test('createPollHealth: warns once after 3 failures in a row, once when it works again', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + const health = createPollHealth('x-alerts'); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 0); + health.failed(fetchFailed('ETIMEDOUT')); + assert.equal(warn.mock.callCount(), 1); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 1, 'silent while it keeps failing'); + health.ok(); + assert.equal(warn.mock.callCount(), 2); + assert.match(warn.mock.calls[1].arguments[1], /works again after 5 failed polls/); + health.ok(); + assert.equal(warn.mock.callCount(), 2, 'a working poll is not logged again'); + assert.equal(error.mock.callCount(), 0); +}); + +test('createPollHealth: a run of failures that ends before the third says nothing', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const health = createPollHealth('x-alerts'); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + health.ok(); + health.failed(fetchFailed()); + health.failed(fetchFailed()); + assert.equal(warn.mock.callCount(), 0); +}); + +// --- ticks ---------------------------------------------------------------------------------- + +function installClient(sent) { + const channel = { + isTextBased: () => true, + permissionsFor: () => ({ has: () => true }), + send: async (payload) => { + sent.push(payload); + return { id: `msg-${sent.length}` }; + }, + }; + const guild = { + id: GUILD, + channels: { cache: new Map([[CH_A, channel]]), fetch: async () => channel }, + members: { me: {} }, + }; + runtime.client = { isReady: () => true, guilds: { cache: new Map([[GUILD, guild]]) } }; + guild.client = runtime.client; +} + +/** + * Run `body` with a fake global fetch. `answer(url, n)` gets the URL and how many requests the + * platform's data endpoints have seen (the token endpoint always works) and returns a Response or throws. + */ +async function withFakeFetch(dataHost, answer, body) { + const realFetch = globalThis.fetch; + const seen = { data: 0 }; + globalThis.fetch = async (url) => { + const u = new URL(String(url)); + if (u.pathname.includes('token')) return Response.json({ access_token: 't', expires_in: 3600 }); + assert.equal(u.hostname, dataHost); + seen.data += 1; + return answer(u, seen.data); + }; + try { + await body(seen); + } finally { + globalThis.fetch = realFetch; + } +} + +const PLATFORMS = [ + { + name: 'twitch', + mod: twitch, + scope: 'twitch-alerts', + host: 'api.twitch.tv', + save: (value) => + setGuildModule(GUILD, 'twitch-alerts', { + enabled: true, + config: { alerts: [{ login: value, channelId: CH_A }] }, + }), + liveAnswer: (u, login) => + Response.json({ + data: u.pathname.endsWith('/users') + ? [] + : [ + { + id: `s-${login}`, + user_login: login, + user_name: login, + title: 'hi', + game_name: 'g', + viewer_count: 3, + started_at: '2026-11-16T00:00:00Z', + thumbnail_url: '', + }, + ], + }), + }, + { + name: 'kick', + mod: kick, + scope: 'kick-alerts', + host: 'api.kick.com', + save: (value) => + setGuildModule(GUILD, 'kick-alerts', { + enabled: true, + config: { alerts: [{ slug: value, channelId: CH_A }] }, + }), + liveAnswer: (u, slug) => + Response.json({ + data: [ + { + slug, + stream_title: 'hi', + category: { name: 'g' }, + stream: { + is_live: true, + start_time: '2026-11-16T00:00:00Z', + viewer_count: 2, + url: `https://kick.com/${slug}`, + }, + }, + ], + }), + }, +]; + +for (const p of PLATFORMS) { + const prep = async (t, value) => { + const sent = []; + installClient(sent); + p.mod.pollHealth.reset(); + const waits = recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await p.save(value); + return { sent, waits, warn, error }; + }; + + test(`${p.name} tick: one network blip is retried silently and the stream is announced`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'blipone'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? Promise.reject(fetchFailed('ECONNRESET')) : p.liveAnswer(u, 'blipone')), + async (seen) => { + await p.mod.tick(); + assert.ok(seen.data >= 2, 'the request was repeated'); + } + ); + assert.equal(sent.length, 1, 'announced as usual'); + assert.deepEqual(waits, [pollRetry.delayMs]); + assert.equal(warn.mock.callCount(), 0); + assert.equal(error.mock.callCount(), 0); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + test(`${p.name} tick: a 503 is retried once too`, async (t) => { + const { sent, warn, error } = await prep(t, 'fivehundred'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? new Response('{}', { status: 503 }) : p.liveAnswer(u, 'fivehundred')), + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: 429 waits for Retry-After before the retry`, async (t) => { + const { sent, waits } = await prep(t, 'slowdown'); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? new Response('{}', { status: 429, headers: { 'Retry-After': '3' } }) + : p.liveAnswer(u, 'slowdown'), + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(waits, [3000]); + assert.equal(sent.length, 1); + }); + + test(`${p.name} tick: 3 failed polls in a row are one warning with the cause code, recovery is one more line`, async (t) => { + const { sent, warn, error } = await prep(t, 'flaky'); + let down = true; + await withFakeFetch( + p.host, + (u) => (down ? Promise.reject(fetchFailed('ECONNRESET')) : p.liveAnswer(u, 'flaky')), + async (seen) => { + await p.mod.tick(); + await p.mod.tick(); + assert.equal(seen.data, 4, 'each failed poll tried twice'); + assert.equal(warn.mock.callCount(), 0, 'one or two failed polls are not logged'); + await p.mod.tick(); + assert.equal(warn.mock.callCount(), 1); + const [scope, line] = warn.mock.calls[0].arguments; + assert.equal(scope, p.scope); + assert.match(line, /3 times in a row/); + assert.match(line, /fetch failed \(ECONNRESET\)/); + assert.doesNotMatch(line, /https?:|tsecret|ksecret|Bearer/); + await p.mod.tick(); + await p.mod.tick(); + assert.equal(warn.mock.callCount(), 1, 'no repeat while it stays down'); + assert.equal(sent.length, 0); + + down = false; + await p.mod.tick(); + } + ); + assert.equal(warn.mock.callCount(), 2); + assert.match(warn.mock.calls[1].arguments[1], /works again after 5 failed polls/); + assert.equal(sent.length, 1, 'the recovered poll announces as usual'); + assert.equal(error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a programming error is still an error, and is not retried`, async (t) => { + const { waits, warn, error } = await prep(t, 'buggy'); + await withFakeFetch( + p.host, + () => Promise.reject(new TypeError("Cannot read properties of undefined (reading 'x')")), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 1, 'not retried'); + } + ); + assert.equal(error.mock.callCount(), 1); + assert.equal(error.mock.calls[0].arguments[0], p.scope); + assert.match(error.mock.calls[0].arguments[2], /Cannot read properties/); + assert.equal(warn.mock.callCount(), 0); + assert.deepEqual(waits, []); + assert.equal(p.mod.pollHealth.failures, 0, 'does not count as a network failure'); + }); + + test(`${p.name} tick: a 4xx answer is not retried and still logged as an error`, async (t) => { + const { waits, warn, error } = await prep(t, 'denied'); + await withFakeFetch( + p.host, + () => new Response('{}', { status: 403 }), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 1); + } + ); + assert.equal(error.mock.callCount(), 1); + assert.match(error.mock.calls[0].arguments[2], /403/); + assert.equal(warn.mock.callCount(), 0); + assert.deepEqual(waits, []); + }); +} + +// --- composition with the batch isolation of rejected values --------------------------------- + +for (const p of PLATFORMS) { + const poison = p.name === 'twitch' ? '_poisonx' : '-poisonx'; + const names = (u) => + p.name === 'twitch' + ? u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login') + : u.searchParams.getAll('slug'); + const saveTwo = (value) => + p.name === 'twitch' + ? setGuildModule(GUILD, 'twitch-alerts', { + enabled: true, + config: { + alerts: [ + { login: poison, channelId: CH_A }, + { login: value, channelId: CH_A }, + ], + }, + }) + : setGuildModule(GUILD, 'kick-alerts', { + enabled: true, + config: { + alerts: [ + { slug: poison, channelId: CH_A }, + { slug: value, channelId: CH_A }, + ], + }, + }); + + test(`${p.name} tick: a blip and a rejected value in the same poll: retried, isolated, victim announced`, async (t) => { + const sent = []; + installClient(sent); + p.mod.pollHealth.reset(); + recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await saveTwo('victimblip'); + await withFakeFetch( + p.host, + (u, n) => { + if (n === 1) return Promise.reject(fetchFailed('ECONNRESET')); + if (names(u).includes(poison)) return new Response('{}', { status: 400 }); + return p.liveAnswer(u, 'victimblip'); + }, + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1, 'the victim is announced'); + assert.equal(error.mock.callCount(), 0); + assert.equal(warn.mock.callCount(), 1, 'only the warning about the rejected value'); + assert.match(warn.mock.calls[0].arguments[1], /rejected 1 of the/); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + test(`${p.name} tick: a network error while values are being isolated fails the poll as a network failure, nothing is quarantined`, async (t) => { + const sent = []; + installClient(sent); + p.mod.pollHealth.reset(); + recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await saveTwo('victimdown'); + const quarantine = p.name === 'twitch' ? p.mod.rejectedLogins : p.mod.rejectedSlugs; + const before = quarantine.size; + await withFakeFetch( + p.host, + (u) => { + if (names(u).length === 2) return new Response('{}', { status: 400 }); + return Promise.reject(fetchFailed('UND_ERR_CONNECT_TIMEOUT')); + }, + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 0); + assert.equal(warn.mock.callCount(), 0, 'one failed poll is not logged'); + assert.equal(p.mod.pollHealth.failures, 1); + assert.equal(quarantine.size, before, 'a network error is not a value fault'); + }); +} + +test('twitch tick: a failed user lookup is neither retried nor logged, the stream is still announced', async (t) => { + const sent = []; + installClient(sent); + twitch.pollHealth.reset(); + const waits = recordSleep(t); + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + await PLATFORMS[0].save('nousers'); + let userCalls = 0; + await withFakeFetch( + 'api.twitch.tv', + (u) => { + if (u.pathname.endsWith('/users')) { + userCalls += 1; + return Promise.reject(fetchFailed('ECONNRESET')); + } + return PLATFORMS[0].liveAnswer(u, 'nousers'); + }, + async () => { + await twitch.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(userCalls, 1); + assert.deepEqual(waits, []); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); +}); + +test.after(() => { + runtime.client = null; +}); From cd6c9b28648acb7e8ed72ce2d2fbf790f32ea91e Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 12:19:02 +0100 Subject: [PATCH 2/3] fix(live-alerts): review follow-ups for the poll error handling Return the tick promise to the scheduler so a slow tick is not started twice. Read Ratelimit-Reset only for a 429. Count only the connection and timeout codes of undici as network errors. Refresh a rejected token and retry once at once. Log an error that is not transient the first time, when it changes and then once an hour, with its stack, and remind hourly during a long network outage. Mark token endpoint errors so batch isolation cannot read a refused token as a rejected value (no more search of ~120 token requests per poll). A network failure of the canary request now fails the poll as that failure and keeps the canary, instead of rethrowing the earlier 400. --- src/modules/kickAlerts.js | 16 +- src/modules/lib/batchIsolation.js | 14 +- src/modules/lib/pollErrors.js | 101 ++++++-- src/modules/twitchAlerts.js | 16 +- test/liveAlertsPollErrors.test.js | 371 +++++++++++++++++++++++++++++- 5 files changed, 472 insertions(+), 46 deletions(-) diff --git a/src/modules/kickAlerts.js b/src/modules/kickAlerts.js index 558d0378..2c8dc5a8 100644 --- a/src/modules/kickAlerts.js +++ b/src/modules/kickAlerts.js @@ -18,7 +18,7 @@ import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveVal import { log } from '../lib/log.js'; import { every } from './schedule.js'; import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; -import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; +import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js'; const API = 'https://api.kick.com/public/v1'; const TOKEN_URL = 'https://id.kick.com/oauth/token'; @@ -71,6 +71,11 @@ export function normaliseKickConfig(raw = {}) { let token = { value: null, expiresAt: 0 }; +/** Forget the cached app token. For tests. */ +export function _resetToken() { + token = { value: null, expiresAt: 0 }; +} + async function appToken() { if (token.value && Date.now() < token.expiresAt - 60_000) return token.value; const body = new URLSearchParams({ @@ -79,7 +84,7 @@ async function appToken() { client_secret: config.kickClientSecret, }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw httpError('Kick token', res); + if (!res.ok) throw tokenError('Kick token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; @@ -102,7 +107,8 @@ function getChannels(slugs) { }); if (res.status === 401) { token = { value: null, expiresAt: 0 }; // force a refresh next call - throw httpError('Kick', res); + // A rejected token: it was just cleared, so one retry at once gets a fresh one. + throw Object.assign(httpError('Kick', res), { retryNow: true }); } if (!res.ok) throw httpError('Kick /channels', res); return (await res.json()).data ?? []; @@ -279,8 +285,6 @@ export async function tick() { every( 'kick-alerts', POLL_MS, - () => { - tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)); - }, + () => tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)), { firstDelayMs: 30_000 } ); diff --git a/src/modules/lib/batchIsolation.js b/src/modules/lib/batchIsolation.js index 7b95ca4d..313cf8cd 100644 --- a/src/modules/lib/batchIsolation.js +++ b/src/modules/lib/batchIsolation.js @@ -31,6 +31,11 @@ // flapping): nothing is quarantined, and when no request of the poll succeeded the poll fails like // any other failed poll. If no value is known to be good yet and nothing succeeded, the poll cannot // tell: nothing is quarantined and it reports `inconclusive`. +// A canary request that fails for a network reason (lib/pollErrors.js isTransient) is not a refusal: +// with nothing succeeded the poll fails with that error, otherwise nothing is quarantined, and the +// canary is kept for the next poll. + +import { isTransient } from './pollErrors.js'; /** Requests per poll, beyond one per batch, for finding the rejected items. */ export const EXTRA_REQUESTS = 120; @@ -218,6 +223,9 @@ export async function fetchIsolated( if (succeeded === 0 && firstFailure !== null && canary) { const answer = await ask([canary], false).catch((err) => err); if (answer !== true) { + // A network failure of the canary says nothing about the values or the canary: the poll + // fails as the network failure it is, and the canary stays. + if (isTransient(answer)) throw answer; quarantine.setGood(null); throw firstFailure; } @@ -264,8 +272,10 @@ export async function fetchIsolated( // poll already fetched. const answer = await ask([canary], false).catch((err) => err); if (answer !== true) { - quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time - if (succeeded === 0) throw found[0].err; + // A network failure (after its retry) is not a refusal of the canary: it stays. + const network = isTransient(answer); + if (!network) quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time + if (succeeded === 0) throw network ? answer : found[0].err; quarantineFound = false; // flapping: some requests answer, some are refused } } else if (succeeded === 0) { diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js index e008bb1d..78a53b2b 100644 --- a/src/modules/lib/pollErrors.js +++ b/src/modules/lib/pollErrors.js @@ -6,9 +6,12 @@ // 429 (honouring Retry-After). 4xx answers other than 429 are not retried: they mean the // request or a value in it is wrong, and asking again gives the same answer. // - createPollHealth() counts consecutive failed polls. One line (warn) after POLL_WARN_AFTER -// failures in a row, one when polling works again, nothing in between. Errors that are not -// network or HTTP problems (a TypeError in our own code, a 4xx that needs a fix such as bad -// credentials) are still logged as errors every time. +// failures in a row, one when polling works again, and a "still failing" reminder once an hour +// while it lasts. Errors that are not network or HTTP problems (a TypeError in our own code, a +// 4xx that needs a fix such as bad credentials) are logged as errors the first time and when +// the message changes, then once an hour ("still failing after N polls"): every error line +// also feeds /health, the error counter and the error reports, so one bad secret must not +// write 1440 of them a day. // "fetch failed", Node's generic network error, hides the reason in `err.cause`; describeError() // puts its code (ECONNRESET, UND_ERR_CONNECT_TIMEOUT, ...) in the text. import { log } from '../../lib/log.js'; @@ -16,6 +19,9 @@ import { log } from '../../lib/log.js'; /** Failed polls in a row before one warning is logged. */ export const POLL_WARN_AFTER = 3; +/** How often a failure that goes on is logged again. */ +export const STILL_FAILING_MS = 60 * 60_000; + /** Retry settings. An object so a test can shorten the wait; do not change it elsewhere. */ export const pollRetry = { delayMs: 3000, @@ -37,6 +43,15 @@ const NETWORK_CODES = new Set([ 'EHOSTUNREACH', 'EHOSTDOWN', 'ERR_SSL_WRONG_VERSION_NUMBER', + // undici: connection and timeout problems only. The others (UND_ERR_INVALID_ARG, ...) are + // faults in how we call it and must stay errors. + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_CLOSED', + 'UND_ERR_ABORTED', + 'UND_ERR_REQ_RETRY', ]); /** The error, its causes, and the members of an AggregateError (a dual-stack connect failure). */ @@ -70,7 +85,7 @@ export function isNetworkError(err) { return chain(err).some( (e) => isTimeoutName(e) || - (typeof e.code === 'string' && (NETWORK_CODES.has(e.code) || e.code.startsWith('UND_ERR_'))) || + (typeof e.code === 'string' && NETWORK_CODES.has(e.code)) || (e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')) ); } @@ -93,9 +108,12 @@ function retryAfterMs(res) { const date = Date.parse(header); if (Number.isFinite(date)) return Math.max(0, date - Date.now()); } - // Twitch states when its rate limit resets, in epoch seconds. - const reset = Number(res.headers?.get?.('ratelimit-reset')); - if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + // Twitch states when its rate limit resets, in epoch seconds. That says nothing about an + // outage, so it is only read for a 429. + if (res.status === 429) { + const reset = Number(res.headers?.get?.('ratelimit-reset')); + if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now()); + } return null; } @@ -113,13 +131,28 @@ export function httpError(label, res) { return err; } -/** Run `fn`; if it fails with a transient error, wait a moment and run it once more. */ +/** + * The error for a non-OK answer of the token endpoint. Its status is kept only when it is + * transient (`httpStatus` always holds it): a refused token is a fault of the request, never of + * one stored value, so batchIsolation's "rejected value" statuses (400, 404, 422) must not see it. + */ +export function tokenError(label, res) { + const err = httpError(label, res); + err.httpStatus = res.status; + if (!isTransientStatus(res.status)) delete err.status; + return err; +} + +/** + * Run `fn`; if it fails with a transient error, wait a moment and run it once more. An error with + * `retryNow` (a 401 whose token was just cleared) is retried at once. + */ export async function withRetry(fn) { try { return await fn(); } catch (err) { - if (!isTransient(err)) throw err; - const wait = err.retryAfterMs ?? pollRetry.delayMs; + if (!isTransient(err) && !err?.retryNow) throw err; + const wait = err.retryNow ? 0 : (err.retryAfterMs ?? pollRetry.delayMs); if (wait > pollRetry.maxWaitMs) throw err; await pollRetry.sleep(wait); return await fn(); @@ -138,39 +171,63 @@ export function describeError(err) { /** * Counts consecutive failed polls of one platform. * @param {string} scope the log scope, e.g. 'twitch-alerts' + * @param {{ warnAfter?: number, repeatMs?: number, now?: () => number }} [o] */ -export function createPollHealth(scope, { warnAfter = POLL_WARN_AFTER } = {}) { - let failures = 0; +export function createPollHealth( + scope, + { warnAfter = POLL_WARN_AFTER, repeatMs = STILL_FAILING_MS, now = () => Date.now() } = {} +) { + let failures = 0; // transient failures in a row let warned = false; + let warnedAt = 0; + /** The error that is not transient and goes on: its text, polls it failed, when it was last logged. */ + let fault = null; + const reset = () => { + failures = 0; + warned = false; + fault = null; + }; return { - /** Failed polls in a row. */ + /** Transient failed polls in a row. */ get failures() { return failures; }, - /** A poll failed with `err`. Logs now, or only after `warnAfter` in a row, see the file head. */ + /** A poll failed with `err`. Logs now, later or not at all, see the file head. */ failed(err) { if (!isTransient(err)) { - log.error(scope, 'poll failed:', describeError(err)); + const text = describeError(err); + if (!fault || fault.text !== text) { + fault = { text, polls: 1, loggedAt: now() }; + log.error(scope, 'poll failed:', err); // the Error itself: its stack reaches the logs + return; + } + fault.polls += 1; + if (now() - fault.loggedAt >= repeatMs) { + fault.loggedAt = now(); + log.error(scope, `poll still failing after ${fault.polls} polls: ${text}`); + } return; } failures += 1; if (failures === warnAfter) { warned = true; + warnedAt = now(); log.warn( scope, `poll failed ${failures} times in a row (${describeError(err)}); new streams may be announced late until it works again` ); + } else if (warned && now() - warnedAt >= repeatMs) { + warnedAt = now(); + log.warn(scope, `poll still failing after ${failures} polls (${describeError(err)})`); } }, /** A poll worked. */ ok() { - if (warned) log.warn(scope, `polling works again after ${failures} failed polls`); - failures = 0; - warned = false; - }, - reset() { - failures = 0; - warned = false; + if (warned || fault) { + log.warn(scope, `polling works again after ${failures + (fault?.polls ?? 0)} failed polls`); + } + reset(); }, + reset, }; } diff --git a/src/modules/twitchAlerts.js b/src/modules/twitchAlerts.js index 816d9da2..7c92499a 100644 --- a/src/modules/twitchAlerts.js +++ b/src/modules/twitchAlerts.js @@ -18,7 +18,7 @@ import { normaliseOnEnd } from '../lib/liveValue.js'; import { log } from '../lib/log.js'; import { every } from './schedule.js'; import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js'; -import { createPollHealth, httpError, withRetry } from './lib/pollErrors.js'; +import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js'; const HELIX = 'https://api.twitch.tv/helix'; const TOKEN_URL = 'https://id.twitch.tv/oauth2/token'; @@ -67,6 +67,11 @@ export function normaliseTwitchConfig(raw = {}) { let token = { value: null, expiresAt: 0 }; +/** Forget the cached app token. For tests. */ +export function _resetToken() { + token = { value: null, expiresAt: 0 }; +} + async function appToken() { if (token.value && Date.now() < token.expiresAt - 60_000) return token.value; const body = new URLSearchParams({ @@ -75,7 +80,7 @@ async function appToken() { grant_type: 'client_credentials', }); const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw httpError('Twitch token', res); + if (!res.ok) throw tokenError('Twitch token', res); const json = await res.json(); token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 }; return token.value; @@ -98,7 +103,8 @@ function helix(path, params, { retry = true } = {}) { }); if (res.status === 401) { token = { value: null, expiresAt: 0 }; // force refresh next call - throw httpError('Twitch', res); + // A rejected token: it was just cleared, so one retry at once gets a fresh one. + throw Object.assign(httpError('Twitch', res), { retryNow: true }); } if (!res.ok) throw httpError(`Twitch ${path}`, res); return (await res.json()).data ?? []; @@ -288,8 +294,6 @@ export async function tick() { every( 'twitch-alerts', POLL_MS, - () => { - tick().catch((err) => log.error('twitch-alerts', 'tick failed:', err.message)); - }, + () => tick().catch((err) => log.error('twitch-alerts', 'tick failed:', err.message)), { firstDelayMs: 25_000 } ); diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js index 41f05cc4..6fca5d5a 100644 --- a/test/liveAlertsPollErrors.test.js +++ b/test/liveAlertsPollErrors.test.js @@ -19,8 +19,11 @@ const { httpError, withRetry, createPollHealth, + tokenError, pollRetry, } = pollErrors; +const { startSchedules, stopSchedules } = await import('../src/modules/schedule.js'); +const { createQuarantine, fetchIsolated } = await import('../src/modules/lib/batchIsolation.js'); const twitch = await import('../src/modules/twitchAlerts.js'); const kick = await import('../src/modules/kickAlerts.js'); @@ -189,6 +192,117 @@ test('createPollHealth: a run of failures that ends before the third says nothin assert.equal(warn.mock.callCount(), 0); }); +test('isNetworkError: only the connection and timeout codes of undici count', () => { + const coded = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error('x'), { code }) }); + for (const code of [ + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + ]) { + assert.equal(isNetworkError(coded(code)), true, code); + } + for (const code of ['UND_ERR_INVALID_ARG', 'UND_ERR_NOT_SUPPORTED', 'UND_ERR_INVALID_RETURN_VALUE']) { + const err = Object.assign(new Error('bad call'), { code }); + assert.equal(isNetworkError(err), false, code); + assert.equal(isTransient(err), false, code); + } +}); + +test('httpError: Ratelimit-Reset is only the wait for a 429, never for a 503', () => { + const reset = String(Math.floor(Date.now() / 1000) + 8); + const wait = httpError('x', fakeResponse(429, { 'ratelimit-reset': reset })).retryAfterMs; + assert.ok(wait > 4000 && wait <= 8000, String(wait)); + assert.equal(httpError('x', fakeResponse(503, { 'ratelimit-reset': reset })).retryAfterMs, undefined); + assert.equal(httpError('x', fakeResponse(503, { 'retry-after': '2' })).retryAfterMs, 2000); + const past = new Date(Date.now() - 60_000).toUTCString(); + assert.equal(httpError('x', fakeResponse(503, { 'retry-after': past })).retryAfterMs, 0); +}); + +test('tokenError: a refused token never carries a status that batch isolation reads as a value fault', () => { + for (const status of [400, 401, 403, 404, 422]) { + const err = tokenError('Twitch token', fakeResponse(status)); + assert.equal(err.status, undefined, String(status)); + assert.equal(err.httpStatus, status); + assert.equal(err.message, `Twitch token ${status}`); + assert.equal(isTransient(err), false); + } + for (const status of [429, 500, 503]) { + const err = tokenError('Twitch token', fakeResponse(status)); + assert.equal(err.status, status); + assert.equal(isTransient(err), true); + } +}); + +test('withRetry: an error marked retryNow is retried at once, but only once', async (t) => { + const waits = recordSleep(t); + let calls = 0; + await withRetry(async () => { + calls += 1; + if (calls === 1) throw Object.assign(httpError('x', fakeResponse(401)), { retryNow: true }); + }); + assert.equal(calls, 2); + assert.deepEqual(waits, [0]); + calls = 0; + await assert.rejects( + withRetry(async () => { + calls += 1; + throw Object.assign(httpError('x', fakeResponse(401)), { retryNow: true }); + }), + /x 401/ + ); + assert.equal(calls, 2); +}); + +test('createPollHealth: an error that is not transient is logged once, again when it changes, then hourly', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + const error = t.mock.method(log, 'error', () => {}); + let now = 1_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + const denied = () => httpError('Twitch /streams', fakeResponse(403)); + const bug = new TypeError("Cannot read properties of undefined (reading 'x')"); + health.failed(bug); + assert.equal(error.mock.callCount(), 1); + assert.equal(error.mock.calls[0].arguments[2], bug, 'the Error itself, so its stack is logged'); + for (let i = 0; i < 100; i += 1) { + now += 60_000; + health.failed(bug); + } + assert.equal(error.mock.callCount(), 2, 'silent for the first hour, then one reminder'); + assert.match(error.mock.calls[1].arguments[1], /still failing after \d+ polls: Cannot read properties/); + health.failed(denied()); // a different message: logged at once + assert.equal(error.mock.callCount(), 3); + now += 61 * 60_000; + health.failed(denied()); + health.failed(denied()); + assert.equal(error.mock.callCount(), 4); + assert.match(error.mock.calls[3].arguments[1], /still failing after 2 polls: Twitch \/streams 403/); + assert.equal(health.failures, 0, 'not a network failure'); + health.ok(); + assert.equal(warn.mock.callCount(), 1); + assert.match(warn.mock.calls[0].arguments[1], /works again after 3 failed polls/); + health.ok(); + assert.equal(warn.mock.callCount(), 1); +}); + +test('createPollHealth: a long network outage gets a reminder every hour', (t) => { + const warn = t.mock.method(log, 'warn', () => {}); + let now = 5_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + for (let i = 0; i < 130; i += 1) { + health.failed(fetchFailed('ENOTFOUND')); + now += 60_000; + } + // 130 minutes: the warning at the 3rd failure, then one reminder per hour + assert.equal(warn.mock.callCount(), 3); + assert.match(warn.mock.calls[0].arguments[1], /3 times in a row/); + assert.match( + warn.mock.calls[1].arguments[1], + /still failing after \d+ polls \(fetch failed \(ENOTFOUND\)\)/ + ); + assert.match(warn.mock.calls[2].arguments[1], /still failing after \d+ polls/); +}); + // --- ticks ---------------------------------------------------------------------------------- function installClient(sent) { @@ -209,16 +323,22 @@ function installClient(sent) { guild.client = runtime.client; } +const tokenOk = () => Response.json({ access_token: 't', expires_in: 3600 }); + /** * Run `body` with a fake global fetch. `answer(url, n)` gets the URL and how many requests the - * platform's data endpoints have seen (the token endpoint always works) and returns a Response or throws. + * platform's data endpoints have seen and returns a Response or throws. `token(n)` answers the n-th + * request of the token endpoint (default: it works). */ -async function withFakeFetch(dataHost, answer, body) { +async function withFakeFetch(dataHost, answer, body, { token } = {}) { const realFetch = globalThis.fetch; - const seen = { data: 0 }; + const seen = { data: 0, token: 0 }; globalThis.fetch = async (url) => { const u = new URL(String(url)); - if (u.pathname.includes('token')) return Response.json({ access_token: 't', expires_in: 3600 }); + if (u.pathname.includes('token')) { + seen.token += 1; + return token ? token(seen.token) : tokenOk(); + } assert.equal(u.hostname, dataHost); seen.data += 1; return answer(u, seen.data); @@ -293,6 +413,7 @@ for (const p of PLATFORMS) { const sent = []; installClient(sent); p.mod.pollHealth.reset(); + p.mod._resetToken(); const waits = recordSleep(t); const warn = t.mock.method(log, 'warn', () => {}); const error = t.mock.method(log, 'error', () => {}); @@ -391,7 +512,8 @@ for (const p of PLATFORMS) { ); assert.equal(error.mock.callCount(), 1); assert.equal(error.mock.calls[0].arguments[0], p.scope); - assert.match(error.mock.calls[0].arguments[2], /Cannot read properties/); + assert.match(error.mock.calls[0].arguments[2].message, /Cannot read properties/); + assert.ok(error.mock.calls[0].arguments[2].stack, 'the Error itself is logged, with its stack'); assert.equal(warn.mock.callCount(), 0); assert.deepEqual(waits, []); assert.equal(p.mod.pollHealth.failures, 0, 'does not count as a network failure'); @@ -408,21 +530,192 @@ for (const p of PLATFORMS) { } ); assert.equal(error.mock.callCount(), 1); - assert.match(error.mock.calls[0].arguments[2], /403/); + assert.match(error.mock.calls[0].arguments[2].message, /403/); assert.equal(warn.mock.callCount(), 0); assert.deepEqual(waits, []); }); + + const tokenBlips = [ + ['a 503', () => new Response('{}', { status: 503 })], + ['a network blip', () => Promise.reject(fetchFailed('ECONNRESET'))], + ]; + for (const [i, [what, first]] of tokenBlips.entries()) { + test(`${p.name} tick: the token endpoint answering with ${what} is retried silently`, async (t) => { + const { sent, waits, warn, error } = await prep(t, `tokenblip${i}`); + await withFakeFetch( + p.host, + (u) => p.liveAnswer(u, `tokenblip${i}`), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 2, 'the token request was repeated'); + }, + { token: (n) => (n === 1 ? first() : tokenOk()) } + ); + assert.equal(sent.length, 1); + assert.deepEqual(waits, [pollRetry.delayMs]); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + } + + test(`${p.name} tick: a token endpoint that refuses with 400 is one error, with no search and no data request`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'badclient'); + await withFakeFetch( + p.host, + (u) => p.liveAnswer(u, 'badclient'), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 1, 'one token request, not one per isolated batch'); + assert.equal(seen.data, 0); + }, + { token: () => new Response('{"error":"invalid client"}', { status: 400 }) } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 1); + assert.match(error.mock.calls[0].arguments[2].message, /token 400/); + assert.equal(warn.mock.callCount(), 0, 'no "every request was refused" warning'); + assert.deepEqual(waits, []); + assert.equal(p.mod.pollHealth.failures, 0); + }); + + const retryAfters = [ + ['seconds', '2', 2000], + ['a date that has passed', new Date(Date.now() - 60_000).toUTCString(), 0], + ]; + for (const [i, [what, header, expected]] of retryAfters.entries()) { + test(`${p.name} tick: a 503 honours Retry-After given as ${what}`, async (t) => { + const { sent, waits, warn, error } = await prep(t, `retryafter${i}`); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? new Response('{}', { status: 503, headers: { 'Retry-After': header } }) + : p.liveAnswer(u, `retryafter${i}`), + async () => { + await p.mod.tick(); + } + ); + assert.deepEqual(waits, [expected]); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + } + + test(`${p.name} tick: a request timeout is retried silently`, async (t) => { + const { sent, warn, error } = await prep(t, 'timedout'); + await withFakeFetch( + p.host, + (u, n) => + n === 1 + ? Promise.reject(new DOMException('The operation was aborted due to timeout', 'TimeoutError')) + : p.liveAnswer(u, 'timedout'), + async () => { + await p.mod.tick(); + } + ); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a rejected token (401) is refreshed and the request retried at once, silently`, async (t) => { + const { sent, waits, warn, error } = await prep(t, 'rotated'); + await withFakeFetch( + p.host, + (u, n) => (n === 1 ? new Response('{}', { status: 401 }) : p.liveAnswer(u, 'rotated')), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.token, 2, 'a fresh token for the retry'); + } + ); + assert.deepEqual(waits, [0]); + assert.equal(sent.length, 1); + assert.equal(warn.mock.callCount() + error.mock.callCount(), 0); + }); + + test(`${p.name} tick: a 401 that stays is an error`, async (t) => { + const { sent, error } = await prep(t, 'norights'); + await withFakeFetch( + p.host, + () => new Response('{}', { status: 401 }), + async (seen) => { + await p.mod.tick(); + assert.equal(seen.data, 2, 'tried once more, not more'); + } + ); + assert.equal(sent.length, 0); + assert.equal(error.mock.callCount(), 1); + }); + + test(`${p.name} tick: bad credentials are one error line for many polls, then one recovery line`, async (t) => { + const { sent, warn, error } = await prep(t, 'badsecret'); + let denied = true; + await withFakeFetch( + p.host, + (u) => (denied ? new Response('{}', { status: 403 }) : p.liveAnswer(u, 'badsecret')), + async () => { + for (let i = 0; i < 5; i += 1) await p.mod.tick(); + assert.equal(error.mock.callCount(), 1, 'logged once, not on every poll'); + denied = false; + await p.mod.tick(); + } + ); + assert.equal(warn.mock.callCount(), 1); + assert.match(warn.mock.calls[0].arguments[1], /works again after 5 failed polls/); + assert.equal(sent.length, 1); + }); } +test('the poll callbacks hand their tick to the scheduler: a tick that is still running is not started twice', async (t) => { + const sent = []; + installClient(sent); + recordSleep(t); + t.mock.method(log, 'warn', () => {}); + t.mock.method(log, 'error', () => {}); + for (const p of PLATFORMS) { + p.mod.pollHealth.reset(); + p.mod._resetToken(); + await p.save('slowtick'); + } + const hits = {}; + const release = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (url) => { + const u = new URL(String(url)); + if (u.pathname.includes('token')) return Promise.resolve(tokenOk()); + hits[u.hostname] = (hits[u.hostname] ?? 0) + 1; + const p = PLATFORMS.find((x) => x.host === u.hostname); + return new Promise((resolve) => release.push(() => resolve(p.liveAnswer(u, 'slowtick')))); + }; + const flush = async () => { + for (let i = 0; i < 30; i += 1) await new Promise((resolve) => setImmediate(resolve)); + }; + t.mock.timers.enable({ apis: ['setTimeout', 'setInterval'] }); + try { + startSchedules(); + t.mock.timers.tick(30_000); // the first ticks (Twitch 25 s, Kick 30 s), now waiting on the network + await flush(); + t.mock.timers.tick(60_000); // the next interval comes while they are still waiting + await flush(); + assert.deepEqual(hits, { 'api.twitch.tv': 1, 'api.kick.com': 1 }, 'no second tick on top of the first'); + for (let i = 0; i < 10 && release.length > 0; i += 1) { + for (const go of release.splice(0)) go(); + await flush(); + } + await stopSchedules(); + } finally { + globalThis.fetch = realFetch; + } +}); + // --- composition with the batch isolation of rejected values --------------------------------- for (const p of PLATFORMS) { - const poison = p.name === 'twitch' ? '_poisonx' : '-poisonx'; + // a different rejected value per test: a quarantined one is left out of the next poll + const poisonFor = (n) => (p.name === 'twitch' ? '_poison' : '-poison') + n; const names = (u) => p.name === 'twitch' ? u.searchParams.getAll(u.pathname.endsWith('/users') ? 'login' : 'user_login') : u.searchParams.getAll('slug'); - const saveTwo = (value) => + const saveTwo = (value, poison) => p.name === 'twitch' ? setGuildModule(GUILD, 'twitch-alerts', { enabled: true, @@ -446,11 +739,13 @@ for (const p of PLATFORMS) { test(`${p.name} tick: a blip and a rejected value in the same poll: retried, isolated, victim announced`, async (t) => { const sent = []; installClient(sent); + const poison = poisonFor('a'); p.mod.pollHealth.reset(); + p.mod._resetToken(); recordSleep(t); const warn = t.mock.method(log, 'warn', () => {}); const error = t.mock.method(log, 'error', () => {}); - await saveTwo('victimblip'); + await saveTwo('victimblip', poison); await withFakeFetch( p.host, (u, n) => { @@ -472,23 +767,40 @@ for (const p of PLATFORMS) { test(`${p.name} tick: a network error while values are being isolated fails the poll as a network failure, nothing is quarantined`, async (t) => { const sent = []; installClient(sent); + const poison = poisonFor('b'); p.mod.pollHealth.reset(); + p.mod._resetToken(); recordSleep(t); const warn = t.mock.method(log, 'warn', () => {}); const error = t.mock.method(log, 'error', () => {}); - await saveTwo('victimdown'); + await saveTwo('victimdown', poison); const quarantine = p.name === 'twitch' ? p.mod.rejectedLogins : p.mod.rejectedSlugs; const before = quarantine.size; + const asked = []; await withFakeFetch( p.host, (u) => { + asked.push(names(u)); + // the value known to be good (the canary) is still answered, so the search starts + if (names(u).length === 1 && names(u)[0] === quarantine.lastGood) + return p.liveAnswer(u, quarantine.lastGood); if (names(u).length === 2) return new Response('{}', { status: 400 }); + if (names(u).includes(poison)) return new Response('{}', { status: 400 }); return Promise.reject(fetchFailed('UND_ERR_CONNECT_TIMEOUT')); }, async () => { await p.mod.tick(); } ); + assert.deepEqual(asked[0].sort(), [poison, 'victimdown'].sort(), 'both values were asked together first'); + assert.ok( + asked.some((names) => names.length === 1 && names[0] === poison), + 'the search got to the rejected value' + ); + assert.ok( + asked.some((names) => names.length === 1 && names[0] === 'victimdown'), + 'and was cut short by the network error' + ); assert.equal(sent.length, 0); assert.equal(error.mock.callCount(), 0); assert.equal(warn.mock.callCount(), 0, 'one failed poll is not logged'); @@ -497,6 +809,45 @@ for (const p of PLATFORMS) { }); } +test('fetchIsolated: a network failure of the canary fails the poll as that failure and keeps the canary', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const refused = (status) => Object.assign(new Error(`refused ${status}`), { status }); + await assert.rejects( + fetchIsolated( + ['a', 'b'], + 2, + async (batch) => { + if (batch[0] === 'known-good') throw fetchFailed('ECONNRESET'); + throw refused(400); + }, + { quarantine } + ), + (err) => err.message === 'fetch failed', + 'not the 400 of the first batch' + ); + assert.equal(quarantine.lastGood, 'known-good', 'the canary was not thrown away'); + assert.equal(quarantine.size, 0); +}); + +test('fetchIsolated: a network failure of the canary while the platform flaps quarantines nothing and keeps the canary', async () => { + const quarantine = createQuarantine(); + quarantine.setGood('known-good'); + const out = await fetchIsolated( + ['fine1', 'flaky'], + 1, + async (batch) => { + if (batch[0] === 'flaky') throw Object.assign(new Error('refused 404'), { status: 404 }); + if (batch[0] === 'known-good') throw fetchFailed('ETIMEDOUT'); + return batch; + }, + { quarantine } + ); + assert.deepEqual(out.results, ['fine1']); + assert.equal(quarantine.size, 0, 'a value is not quarantined on the word of a failing request'); + assert.ok(out.unresolved.has('flaky')); +}); + test('twitch tick: a failed user lookup is neither retried nor logged, the stream is still announced', async (t) => { const sent = []; installClient(sent); From 40d924cb5f01331adb20beb725447352d0671853 Mon Sep 17 00:00:00 2001 From: Ferdinand99 Date: Mon, 16 Nov 2026 12:41:15 +0100 Subject: [PATCH 3/3] fix(live-alerts): classify fetch failures by their cause code and key the error throttle by kind A TypeError('fetch failed') with a cause is now a network error only when the cause code is a connection or timeout code; UND_ERR_INVALID_ARG and similar stay errors. The hourly throttle for errors that are not transient keys on name, status and the message without numbers, so alternating errors or a changing value in the message are not logged on every poll. --- src/modules/lib/pollErrors.js | 43 +++++++++++++++++++++---------- test/liveAlertsPollErrors.test.js | 43 +++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/modules/lib/pollErrors.js b/src/modules/lib/pollErrors.js index 78a53b2b..e45633f3 100644 --- a/src/modules/lib/pollErrors.js +++ b/src/modules/lib/pollErrors.js @@ -82,11 +82,15 @@ export function causeCode(err) { export function isNetworkError(err) { if (!err || typeof err !== 'object') return false; if (typeof err.status === 'number') return false; // we got an answer - return chain(err).some( - (e) => - isTimeoutName(e) || - (typeof e.code === 'string' && NETWORK_CODES.has(e.code)) || - (e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')) + const items = chain(err); + if (items.some(isTimeoutName)) return true; + // undici wraps every dispatcher error in TypeError('fetch failed', { cause }): when a cause + // carries a code, the code decides (UND_ERR_INVALID_ARG is our fault, ECONNRESET is the network). + const codes = items.map((e) => e.code).filter((c) => typeof c === 'string' && c); + if (codes.length > 0) return codes.some((c) => NETWORK_CODES.has(c)); + // No code anywhere: the generic wrapper alone counts as a network failure. + return items.some( + (e) => e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated') ); } @@ -180,12 +184,21 @@ export function createPollHealth( let failures = 0; // transient failures in a row let warned = false; let warnedAt = 0; - /** The error that is not transient and goes on: its text, polls it failed, when it was last logged. */ - let fault = null; + /** + * The errors that are not transient and go on, by kind (name, status, message without numbers), + * so errors that alternate, or whose message holds a changing value, are not logged every poll. + * A fixed number of kinds is kept; the oldest is dropped. + * @type {Map} + */ + const faults = new Map(); + const MAX_KINDS = 10; + const kindOf = (err) => + [err?.name, err?.httpStatus ?? err?.status ?? '', describeError(err).replace(/\d+/g, '#')].join('|'); + const faultPolls = () => [...faults.values()].reduce((sum, f) => sum + f.polls, 0); const reset = () => { failures = 0; warned = false; - fault = null; + faults.clear(); }; return { /** Transient failed polls in a row. */ @@ -195,16 +208,18 @@ export function createPollHealth( /** A poll failed with `err`. Logs now, later or not at all, see the file head. */ failed(err) { if (!isTransient(err)) { - const text = describeError(err); - if (!fault || fault.text !== text) { - fault = { text, polls: 1, loggedAt: now() }; + const kind = kindOf(err); + const fault = faults.get(kind); + if (!fault) { + if (faults.size >= MAX_KINDS) faults.delete(faults.keys().next().value); + faults.set(kind, { polls: 1, loggedAt: now() }); log.error(scope, 'poll failed:', err); // the Error itself: its stack reaches the logs return; } fault.polls += 1; if (now() - fault.loggedAt >= repeatMs) { fault.loggedAt = now(); - log.error(scope, `poll still failing after ${fault.polls} polls: ${text}`); + log.error(scope, `poll still failing after ${fault.polls} polls: ${describeError(err)}`); } return; } @@ -223,8 +238,8 @@ export function createPollHealth( }, /** A poll worked. */ ok() { - if (warned || fault) { - log.warn(scope, `polling works again after ${failures + (fault?.polls ?? 0)} failed polls`); + if (warned || faults.size > 0) { + log.warn(scope, `polling works again after ${failures + faultPolls()} failed polls`); } reset(); }, diff --git a/test/liveAlertsPollErrors.test.js b/test/liveAlertsPollErrors.test.js index 6fca5d5a..89521c23 100644 --- a/test/liveAlertsPollErrors.test.js +++ b/test/liveAlertsPollErrors.test.js @@ -209,6 +209,22 @@ test('isNetworkError: only the connection and timeout codes of undici count', () } }); +test('isNetworkError: a fetch failure with a cause is classified by the cause code', () => { + const wrapped = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error('x'), { code }) }); + for (const code of ['ECONNRESET', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'ENOTFOUND']) { + assert.equal(isNetworkError(wrapped(code)), true, code); + assert.equal(isTransient(wrapped(code)), true, code); + } + for (const code of ['UND_ERR_INVALID_ARG', 'UND_ERR_NOT_SUPPORTED', 'UND_ERR_INVALID_RETURN_VALUE']) { + assert.equal(isNetworkError(wrapped(code)), false, code); + assert.equal(isTransient(wrapped(code)), false, code); + } + // the wrapper alone, or with a cause that has no code, still counts as a network failure + assert.equal(isNetworkError(new TypeError('fetch failed')), true); + assert.equal(isNetworkError(new TypeError('fetch failed', { cause: new Error('no code') })), true); + assert.equal(isNetworkError(new TypeError('Cannot read properties of undefined')), false); +}); + test('httpError: Ratelimit-Reset is only the wait for a 429, never for a 503', () => { const reset = String(Math.floor(Date.now() / 1000) + 8); const wait = httpError('x', fakeResponse(429, { 'ratelimit-reset': reset })).retryAfterMs; @@ -280,11 +296,33 @@ test('createPollHealth: an error that is not transient is logged once, again whe assert.equal(health.failures, 0, 'not a network failure'); health.ok(); assert.equal(warn.mock.callCount(), 1); - assert.match(warn.mock.calls[0].arguments[1], /works again after 3 failed polls/); + assert.match(warn.mock.calls[0].arguments[1], /works again after 104 failed polls/); health.ok(); assert.equal(warn.mock.callCount(), 1); }); +test('createPollHealth: alternating errors, or a message with a changing number, are not logged on every poll', (t) => { + const error = t.mock.method(log, 'error', () => {}); + let now = 9_000_000; + const health = createPollHealth('x-alerts', { now: () => now }); + const a = () => httpError('Twitch /streams', fakeResponse(403)); + const b = () => httpError('Twitch token', fakeResponse(401)); + for (let i = 0; i < 20; i += 1) { + health.failed(a()); + health.failed(b()); + now += 60_000; + } + assert.equal(error.mock.callCount(), 2, 'one line for each of the two kinds'); + for (let i = 0; i < 30; i += 1) { + health.failed(new RangeError(`index ${i} is out of range 0..${100 + i}`)); + now += 60_000; + } + assert.equal(error.mock.callCount(), 3, 'the number in the message does not make a new kind'); + // a different status is a different kind + health.failed(httpError('Twitch /streams', fakeResponse(404))); + assert.equal(error.mock.callCount(), 4); +}); + test('createPollHealth: a long network outage gets a reminder every hour', (t) => { const warn = t.mock.method(log, 'warn', () => {}); let now = 5_000_000; @@ -830,7 +868,7 @@ test('fetchIsolated: a network failure of the canary fails the poll as that fail assert.equal(quarantine.size, 0); }); -test('fetchIsolated: a network failure of the canary while the platform flaps quarantines nothing and keeps the canary', async () => { +test('fetchIsolated: a network failure of the canary while the platform flaps quarantines nothing and remembers the value for the next poll', async () => { const quarantine = createQuarantine(); quarantine.setGood('known-good'); const out = await fetchIsolated( @@ -846,6 +884,7 @@ test('fetchIsolated: a network failure of the canary while the platform flaps qu assert.deepEqual(out.results, ['fine1']); assert.equal(quarantine.size, 0, 'a value is not quarantined on the word of a failing request'); assert.ok(out.unresolved.has('flaky')); + assert.ok(quarantine.suspects.has('flaky'), 'it is batched apart from the clean values next time'); }); test('twitch tick: a failed user lookup is neither retried nor logged, the stream is still announced', async (t) => {