Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 33 additions & 18 deletions src/modules/kickAlerts.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { encodeLiveValue, decodeLiveValue, normaliseOnEnd } from '../lib/liveVal
import { log } from '../lib/log.js';
import { every } from './schedule.js';
import { createQuarantine, fetchIsolated, rejectedWarning } from './lib/batchIsolation.js';
import { createPollHealth, httpError, tokenError, withRetry } from './lib/pollErrors.js';

const API = 'https://api.kick.com/public/v1';
const TOKEN_URL = 'https://id.kick.com/oauth/token';
Expand Down Expand Up @@ -70,6 +71,11 @@ export function normaliseKickConfig(raw = {}) {

let token = { value: null, expiresAt: 0 };

/** Forget the cached app token. For tests. */
export function _resetToken() {
token = { value: null, expiresAt: 0 };
}

async function appToken() {
if (token.value && Date.now() < token.expiresAt - 60_000) return token.value;
const body = new URLSearchParams({
Expand All @@ -78,28 +84,35 @@ async function appToken() {
client_secret: config.kickClientSecret,
});
const res = await fetch(TOKEN_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) });
if (!res.ok) throw new Error(`Kick token ${res.status}`);
if (!res.ok) throw tokenError('Kick token', res);
const json = await res.json();
token = { value: json.access_token, expiresAt: Date.now() + (json.expires_in ?? 3600) * 1000 };
return token.value;
}

async function getChannels(slugs) {
/**
* One channels request. A network error, 5xx or 429 is retried once (see lib/pollErrors.js); the
* retry sits inside the request, so fetchIsolated only sees an error once the retry failed too.
*/
function getChannels(slugs) {
const url = new URL(`${API}/channels`);
for (const s of slugs) url.searchParams.append('slug', s);
const res = await fetch(url, {
headers: {
Authorization: `Bearer ${await appToken()}`,
'Content-Type': 'application/json',
},
signal: AbortSignal.timeout(10_000),
return withRetry(async () => {
const res = await fetch(url, {
headers: {
Authorization: `Bearer ${await appToken()}`,
'Content-Type': 'application/json',
},
signal: AbortSignal.timeout(10_000),
});
if (res.status === 401) {
token = { value: null, expiresAt: 0 }; // force a refresh next call
// A rejected token: it was just cleared, so one retry at once gets a fresh one.
throw Object.assign(httpError('Kick', res), { retryNow: true });
}
if (!res.ok) throw httpError('Kick /channels', res);
return (await res.json()).data ?? [];
});
if (res.status === 401) {
token = { value: null, expiresAt: 0 }; // force a refresh next call
throw new Error('Kick 401');
}
if (!res.ok) throw Object.assign(new Error(`Kick /channels ${res.status}`), { status: res.status });
return (await res.json()).data ?? [];
}

/**
Expand Down Expand Up @@ -183,6 +196,9 @@ export function buildPayload(channel, alert) {

// --- poll loop -------------------------------------------------------

/** Consecutive failed polls: logged once after a few in a row, see lib/pollErrors.js. */
export const pollHealth = createPollHealth('kick-alerts');

/** One poll; exported for tests. */
export async function tick() {
if (!allClientsReady() || !config.kickEnabled) return;
Expand Down Expand Up @@ -230,9 +246,10 @@ export async function tick() {
},
});
} catch (err) {
log.error('kick-alerts', 'poll failed:', err.message);
pollHealth.failed(err);
return;
}
pollHealth.ok();

for (const { guildId, alert } of jobs) {
// Not looked up this poll (rejected by Kick, waiting out a backoff, or over the request
Expand Down Expand Up @@ -268,8 +285,6 @@ export async function tick() {
every(
'kick-alerts',
POLL_MS,
() => {
tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message));
},
() => tick().catch((err) => log.error('kick-alerts', 'tick failed:', err.message)),
{ firstDelayMs: 30_000 }
);
14 changes: 12 additions & 2 deletions src/modules/lib/batchIsolation.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@
// flapping): nothing is quarantined, and when no request of the poll succeeded the poll fails like
// any other failed poll. If no value is known to be good yet and nothing succeeded, the poll cannot
// tell: nothing is quarantined and it reports `inconclusive`.
// A canary request that fails for a network reason (lib/pollErrors.js isTransient) is not a refusal:
// with nothing succeeded the poll fails with that error, otherwise nothing is quarantined, and the
// canary is kept for the next poll.

import { isTransient } from './pollErrors.js';

/** Requests per poll, beyond one per batch, for finding the rejected items. */
export const EXTRA_REQUESTS = 120;
Expand Down Expand Up @@ -218,6 +223,9 @@ export async function fetchIsolated(
if (succeeded === 0 && firstFailure !== null && canary) {
const answer = await ask([canary], false).catch((err) => err);
if (answer !== true) {
// A network failure of the canary says nothing about the values or the canary: the poll
// fails as the network failure it is, and the canary stays.
if (isTransient(answer)) throw answer;
quarantine.setGood(null);
throw firstFailure;
}
Expand Down Expand Up @@ -264,8 +272,10 @@ export async function fetchIsolated(
// poll already fetched.
const answer = await ask([canary], false).catch((err) => err);
if (answer !== true) {
quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time
if (succeeded === 0) throw found[0].err;
// A network failure (after its retry) is not a refusal of the canary: it stays.
const network = isTransient(answer);
if (!network) quarantine.setGood(null); // it may have gone bad itself: pick a fresh one next time
if (succeeded === 0) throw network ? answer : found[0].err;
quarantineFound = false; // flapping: some requests answer, some are refused
}
} else if (succeeded === 0) {
Expand Down
248 changes: 248 additions & 0 deletions src/modules/lib/pollErrors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
// Transient errors in the shared live-alert polls (Twitch and Kick).
//
// A poll asks a platform for every saved streamer at once, once a minute. A network blip (a reset
// connection, a DNS hiccup, a 502) should not page the operator, so:
// - withRetry() repeats a failed request once after a short delay, for network errors, 5xx and
// 429 (honouring Retry-After). 4xx answers other than 429 are not retried: they mean the
// request or a value in it is wrong, and asking again gives the same answer.
// - createPollHealth() counts consecutive failed polls. One line (warn) after POLL_WARN_AFTER
// failures in a row, one when polling works again, and a "still failing" reminder once an hour
// while it lasts. Errors that are not network or HTTP problems (a TypeError in our own code, a
// 4xx that needs a fix such as bad credentials) are logged as errors the first time and when
// the message changes, then once an hour ("still failing after N polls"): every error line
// also feeds /health, the error counter and the error reports, so one bad secret must not
// write 1440 of them a day.
// "fetch failed", Node's generic network error, hides the reason in `err.cause`; describeError()
// puts its code (ECONNRESET, UND_ERR_CONNECT_TIMEOUT, ...) in the text.
import { log } from '../../lib/log.js';

/** Failed polls in a row before one warning is logged. */
export const POLL_WARN_AFTER = 3;

/** How often a failure that goes on is logged again. */
export const STILL_FAILING_MS = 60 * 60_000;

/** Retry settings. An object so a test can shorten the wait; do not change it elsewhere. */
export const pollRetry = {
delayMs: 3000,
/** A longer Retry-After than this is not waited for: the poll fails and the next one tries again. */
maxWaitMs: 15_000,
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
};

const NETWORK_CODES = new Set([
'ECONNRESET',
'ECONNREFUSED',
'ECONNABORTED',
'ETIMEDOUT',
'ENOTFOUND',
'EAI_AGAIN',
'EPIPE',
'ENETUNREACH',
'ENETDOWN',
'EHOSTUNREACH',
'EHOSTDOWN',
'ERR_SSL_WRONG_VERSION_NUMBER',
// undici: connection and timeout problems only. The others (UND_ERR_INVALID_ARG, ...) are
// faults in how we call it and must stay errors.
'UND_ERR_SOCKET',
'UND_ERR_CONNECT_TIMEOUT',
'UND_ERR_HEADERS_TIMEOUT',
'UND_ERR_BODY_TIMEOUT',
'UND_ERR_CLOSED',
'UND_ERR_ABORTED',
'UND_ERR_REQ_RETRY',
]);

/** The error, its causes, and the members of an AggregateError (a dual-stack connect failure). */
function chain(err) {
const out = [];
const queue = [err];
while (queue.length > 0 && out.length < 8) {
const e = queue.shift();
if (!e || typeof e !== 'object' || out.includes(e)) continue;
out.push(e);
queue.push(e.cause);
if (Array.isArray(e.errors)) queue.push(...e.errors);
}
return out;
}

const isTimeoutName = (e) => e?.name === 'TimeoutError' || e?.name === 'AbortError';

/** The first string error code in the chain (ECONNRESET, UND_ERR_SOCKET, ...), or the timeout name. */
export function causeCode(err) {
const items = chain(err);
for (const e of items) if (typeof e.code === 'string' && e.code) return e.code;
const named = items.find(isTimeoutName);
return named ? named.name : null;
}

/** A failed request that never got an HTTP answer: connection, DNS, TLS, timeout, broken body. */
export function isNetworkError(err) {
if (!err || typeof err !== 'object') return false;
if (typeof err.status === 'number') return false; // we got an answer
const items = chain(err);
if (items.some(isTimeoutName)) return true;
// undici wraps every dispatcher error in TypeError('fetch failed', { cause }): when a cause
// carries a code, the code decides (UND_ERR_INVALID_ARG is our fault, ECONNRESET is the network).
const codes = items.map((e) => e.code).filter((c) => typeof c === 'string' && c);
if (codes.length > 0) return codes.some((c) => NETWORK_CODES.has(c));
// No code anywhere: the generic wrapper alone counts as a network failure.
return items.some(
(e) => e instanceof TypeError && (e.message === 'fetch failed' || e.message === 'terminated')
);
}

/** 5xx and 429: the platform is busy or down, try again later. Other statuses are not transient. */
export function isTransientStatus(status) {
return typeof status === 'number' && (status >= 500 || status === 429);
}

/** Retry once, wait a moment, and count on the next poll: network errors, 5xx and 429. */
export function isTransient(err) {
return isNetworkError(err) || isTransientStatus(err?.status);
}

function retryAfterMs(res) {
const header = res.headers?.get?.('retry-after');
if (header) {
const seconds = Number(header);
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
const date = Date.parse(header);
if (Number.isFinite(date)) return Math.max(0, date - Date.now());
}
// Twitch states when its rate limit resets, in epoch seconds. That says nothing about an
// outage, so it is only read for a 429.
if (res.status === 429) {
const reset = Number(res.headers?.get?.('ratelimit-reset'));
if (Number.isFinite(reset) && reset > 0) return Math.max(0, reset * 1000 - Date.now());
}
return null;
}

/**
* The error for a non-OK answer: `${label} ${status}`, with `status`, and `retryAfterMs` when the
* platform said how long to wait (429 and 503 only). The message holds no URL, so no credentials.
*/
export function httpError(label, res) {
const err = new Error(`${label} ${res.status}`);
err.status = res.status;
if (res.status === 429 || res.status === 503) {
const wait = retryAfterMs(res);
if (wait !== null) err.retryAfterMs = wait;
}
return err;
}

/**
* The error for a non-OK answer of the token endpoint. Its status is kept only when it is
* transient (`httpStatus` always holds it): a refused token is a fault of the request, never of
* one stored value, so batchIsolation's "rejected value" statuses (400, 404, 422) must not see it.
*/
export function tokenError(label, res) {
const err = httpError(label, res);
err.httpStatus = res.status;
if (!isTransientStatus(res.status)) delete err.status;
return err;
}

/**
* Run `fn`; if it fails with a transient error, wait a moment and run it once more. An error with
* `retryNow` (a 401 whose token was just cleared) is retried at once.
*/
export async function withRetry(fn) {
try {
return await fn();
} catch (err) {
if (!isTransient(err) && !err?.retryNow) throw err;
const wait = err.retryNow ? 0 : (err.retryAfterMs ?? pollRetry.delayMs);
if (wait > pollRetry.maxWaitMs) throw err;
await pollRetry.sleep(wait);
return await fn();
}
}

/** Short text for a log line: the message, plus the cause code when the message hides it. */
export function describeError(err) {
const message = String(err?.message ?? err)
.replace(/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi, '<url>') // never a URL, it may carry credentials
.slice(0, 200);
const code = causeCode(err);
return code && !message.includes(code) ? `${message} (${code})` : message;
}

/**
* Counts consecutive failed polls of one platform.
* @param {string} scope the log scope, e.g. 'twitch-alerts'
* @param {{ warnAfter?: number, repeatMs?: number, now?: () => number }} [o]
*/
export function createPollHealth(
scope,
{ warnAfter = POLL_WARN_AFTER, repeatMs = STILL_FAILING_MS, now = () => Date.now() } = {}
) {
let failures = 0; // transient failures in a row
let warned = false;
let warnedAt = 0;
/**
* The errors that are not transient and go on, by kind (name, status, message without numbers),
* so errors that alternate, or whose message holds a changing value, are not logged every poll.
* A fixed number of kinds is kept; the oldest is dropped.
* @type {Map<string, { polls: number, loggedAt: number }>}
*/
const faults = new Map();
const MAX_KINDS = 10;
const kindOf = (err) =>
[err?.name, err?.httpStatus ?? err?.status ?? '', describeError(err).replace(/\d+/g, '#')].join('|');
const faultPolls = () => [...faults.values()].reduce((sum, f) => sum + f.polls, 0);
const reset = () => {
failures = 0;
warned = false;
faults.clear();
};
return {
/** Transient failed polls in a row. */
get failures() {
return failures;
},
/** A poll failed with `err`. Logs now, later or not at all, see the file head. */
failed(err) {
if (!isTransient(err)) {
const kind = kindOf(err);
const fault = faults.get(kind);
if (!fault) {
if (faults.size >= MAX_KINDS) faults.delete(faults.keys().next().value);
faults.set(kind, { polls: 1, loggedAt: now() });
log.error(scope, 'poll failed:', err); // the Error itself: its stack reaches the logs
return;
}
fault.polls += 1;
if (now() - fault.loggedAt >= repeatMs) {
fault.loggedAt = now();
log.error(scope, `poll still failing after ${fault.polls} polls: ${describeError(err)}`);
}
return;
}
failures += 1;
if (failures === warnAfter) {
warned = true;
warnedAt = now();
log.warn(
scope,
`poll failed ${failures} times in a row (${describeError(err)}); new streams may be announced late until it works again`
);
} else if (warned && now() - warnedAt >= repeatMs) {
warnedAt = now();
log.warn(scope, `poll still failing after ${failures} polls (${describeError(err)})`);
}
},
/** A poll worked. */
ok() {
if (warned || faults.size > 0) {
log.warn(scope, `polling works again after ${failures + faultPolls()} failed polls`);
}
reset();
},
reset,
};
}
Loading
Loading