diff --git a/CHANGELOG.md b/CHANGELOG.md index 5701d56a..542025bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # node-pre-gyp changelog ## master +- Retry binary downloads on transient failures (HTTP 429/5xx and connection-level errors) with exponential backoff and full jitter. Configurable via `--retries` (default 2), `--retry_delay` (default 1000ms) and `--timeout` (default 30000ms). +- HTTP download errors now carry `statusCode`, which restores the intended `Tried to download()` diagnostic in the fallback message. Previously every HTTP failure reported the generic "Pre-built binaries not installable" / "Hit error" text. ## 2.0.4-pre.0 - Test the release workflow diff --git a/README.md b/README.md index d7b36e78..fd2d133b 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,13 @@ Options include: - `--target_arch=ia32`: Pass the target arch and override the host `arch`. Any value that is [supported by Node.js](https://nodejs.org/api/os.html#osarch) is valid. - `--target_platform=win32`: Pass the target platform and override the host `platform`. Valid values are `linux`, `darwin`, `win32`, `sunos`, `freebsd`, `openbsd`, and `aix`. - `--acl=`: Set the S3 ACL when publishing binaries (e.g., `public-read`, `private`). Overrides the `binary.acl` setting in package.json. + - `--retries=`: Number of times to retry a failed binary download, after the first attempt (default: `2`). Pass `--retries=0` to disable retrying and fall back to a source build immediately. + - `--retry_delay=`: Base delay in milliseconds for the exponential backoff between download retries (default: `1000`). + - `--timeout=`: Per-attempt timeout in milliseconds for a binary download request (default: `30000`). This applies to the response headers, not the body transfer, so a large binary downloading slowly is not interrupted. Pass `--timeout=0` to disable it. + +Downloads are retried on transient failures only: HTTP `429` and `5xx` responses, and connection-level errors such as `ECONNRESET` or a socket hang up. A `404` (no pre-built binary for this platform and ABI) and a `403` (private binary, handled by the authenticated download path) are never retried, so they still fall back to a source build immediately. Backoff uses full jitter, which spreads out retries when many packages are installed in parallel. + +These three options can also be set as npm config, in which case they take the `node_pre_gyp_` prefix — for example `npm config set node_pre_gyp_retries 5`, or `node_pre_gyp_retries=5 npm install`. Both `--build-from-source` and `--fallback-to-build` can be passed alone or they can provide values. You can pass `--fallback-to-build=false` to override the option as declared in package.json. In addition to being able to pass `--build-from-source` you can also pass `--build-from-source=myapp` where `myapp` is the name of your module. diff --git a/lib/install.js b/lib/install.js index 82705428..b7131c37 100644 --- a/lib/install.js +++ b/lib/install.js @@ -25,6 +25,158 @@ try { // do nothing } +const DEFAULT_RETRIES = 2; +const DEFAULT_RETRY_DELAY = 1000; +const MAX_RETRY_DELAY = 30000; +const DEFAULT_TIMEOUT = 30000; + +/** + * Network error codes worth retrying. node-fetch v2 wraps system errors in a FetchError that copies `code` off the + * underlying error. ENOTFOUND is deliberately excluded: a DNS miss is almost always a misconfigured binary.host, so + * retrying it only delays the fallback-to-build. EAI_AGAIN is included because it is explicitly a temporary failure. + */ +const RETRYABLE_NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'EPIPE', + 'EAI_AGAIN', + 'ENETUNREACH', + 'EHOSTUNREACH' +]); + +/** + * Determines whether an HTTP status is worth retrying. 429 is rate limiting and 5xx is server-side, both transient. + * + * @param {number} status HTTP status code from the response. + * @returns {boolean} True if the request should be retried. + */ +function is_retryable_status(status) { + return status === 429 || (status >= 500 && status <= 599); +} + +/** + * Determines whether a thrown request error is transient. Matches node-fetch's timeout types, the known transient + * system codes, and a bare socket hangup, which node-fetch surfaces with no usable code. + * + * @param {Error} err Error thrown by fetch, typically a node-fetch FetchError. + * @returns {boolean} True if the request should be retried. + */ +function is_retryable_error(err) { + if (!err) return false; + if (err.name === 'AbortError') return false; + if (err.type === 'request-timeout' || err.type === 'body-timeout') return true; + if (RETRYABLE_NETWORK_CODES.has(err.code)) return true; + if (typeof err.message === 'string' && err.message.includes('socket hang up')) return true; + return false; +} + +/** + * Calculates an exponential backoff delay using full jitter, so the sleep is uniform in [0, base * 2^attempt] up to + * MAX_RETRY_DELAY. + * + * @param {number} attempt Zero-based index of the retry about to be made. + * @param {number} baseDelay Base delay in milliseconds. + * @returns {number} Delay in milliseconds to wait before the next attempt. + */ +function backoff_delay(attempt, baseDelay) { + const ceiling = Math.min(baseDelay * Math.pow(2, attempt), MAX_RETRY_DELAY); + return Math.floor(Math.random() * ceiling); +} + +/** + * Promisified setTimeout. + * + * @param {number} ms Milliseconds to wait. + * @returns {Promise} Resolves once the delay has elapsed. + */ +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Resolves download retry settings, mirroring how proxy and cafile are sourced: an explicit option wins, then the + * npm-inherited config, then the default. + * + * Values arriving from npm_config_* are arbitrary strings, so each is range-checked after parsing. An unparseable or + * negative value falls back to its default. Note that 0 is meaningful and is honoured for all three: no retries, no + * backoff, and, following node-fetch's own convention for `timeout`, no timeout. + * + * @param {Object} opts Options object as built by versioning.evaluate and topped up by install(). + * @returns {{retries: number, retryDelay: number, timeout: number}} Validated retry settings. + */ +function resolve_retry_opts(opts) { + const pick = (...vals) => vals.find((v) => v !== undefined && v !== null && v !== ''); + + const rawRetries = pick(opts.retries, opts.node_pre_gyp_retries, process.env.node_pre_gyp_retries); + const rawDelay = pick(opts.retry_delay, opts.node_pre_gyp_retry_delay, process.env.node_pre_gyp_retry_delay); + const rawTimeout = pick(opts.timeout, opts.node_pre_gyp_timeout, process.env.node_pre_gyp_timeout); + + let retries = rawRetries === undefined ? DEFAULT_RETRIES : parseInt(rawRetries, 10); + let retryDelay = rawDelay === undefined ? DEFAULT_RETRY_DELAY : parseInt(rawDelay, 10); + let timeout = rawTimeout === undefined ? DEFAULT_TIMEOUT : parseInt(rawTimeout, 10); + + if (!Number.isFinite(retries) || retries < 0) retries = DEFAULT_RETRIES; + if (!Number.isFinite(retryDelay) || retryDelay < 0) retryDelay = DEFAULT_RETRY_DELAY; + if (!Number.isFinite(timeout) || timeout < 0) timeout = DEFAULT_TIMEOUT; + + return { retries, retryDelay, timeout }; +} + +/** + * Performs the binary download request with bounded retries and exponential backoff. + * + * Only the request and its status are retried, never the body stream: once the response is handed back and piped into + * tar, partial files exist on disk and a second attempt would extract over a half-written tree. See place_binary. + * + * @param {string} sanitized URL to download from. + * @param {Object} fetchOpts Options passed through to fetch, such as agent and timeout. + * @param {{retries: number, retryDelay: number}} retryOpts Retry settings from resolve_retry_opts. + * @returns {Promise} Resolves with the node-fetch Response. + * @throws {Error} The last error encountered once attempts are exhausted; carries statusCode for HTTP failures. + */ +async function fetch_with_retry(sanitized, fetchOpts, retryOpts) { + const { retries, retryDelay } = retryOpts; + let lastError; + + for (let attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + const delay = backoff_delay(attempt - 1, retryDelay); + log.warn('install', + `retrying download (attempt ${attempt + 1}/${retries + 1}) in ${delay}ms: ${lastError}`); + await sleep(delay); + } + + let res; + try { + res = await fetch(sanitized, fetchOpts); + } catch (e) { + if (attempt < retries && is_retryable_error(e)) { + lastError = e.message; + continue; + } + throw e; + } + + if (res.ok || !is_retryable_status(res.status)) { + return res; + } + + // node-fetch keeps the socket checked out until the body is consumed, so drain it before retrying. + if (res.body && typeof res.body.resume === 'function') res.body.resume(); + + lastError = `response status ${res.status} ${res.statusText}`; + if (attempt >= retries) { + const err = new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`); + err.statusCode = res.status; + throw err; + } + } + + // Unreachable: every path above either returns or throws. Guards against a future edit breaking that invariant. + throw new Error(`download failed after ${retries + 1} attempts: ${lastError}`); +} + function place_binary_authenticated(opts, targetDir, callback) { log.info('install', 'Attempting authenticated S3 download'); @@ -121,7 +273,11 @@ function place_binary(uri, targetDir, opts, callback) { log.log('download', `proxy agent configured using: "${proxyUrl}"`); } - fetch(sanitized, { agent }) + const retryOpts = resolve_retry_opts(opts); + + // The timeout covers the response headers only because the body is piped rather than buffered, so a slow but + // progressing tarball download is never killed mid-stream. Calling res.buffer()/text() here would change that. + fetch_with_retry(sanitized, { agent, timeout: retryOpts.timeout }, retryOpts) .then((res) => { if (!res.ok) { // If we get 403 Forbidden, the binary might be private - try authenticated download @@ -132,8 +288,14 @@ function place_binary(uri, targetDir, opts, callback) { place_binary_authenticated(opts, targetDir, callback); return { authenticated: true }; } - throw new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`); + const err = new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`); + // print_fallback_error branches on statusCode for its "Tried to download(N)" diagnostic. + err.statusCode = res.status; + throw err; } + + // Committed from here: the body streams into tar, so a mid-stream failure leaves partial files in + // targetDir and is not safe to retry without extracting to a temp dir first. const dataStream = res.body; return new Promise((resolve, reject) => { @@ -265,6 +427,14 @@ function install(gyp, argv, callback) { opts.ca = gyp.opts.ca; opts.cafile = gyp.opts.cafile; + // versioning.evaluate() builds a fresh opts object and drops anything it does not read, so these must be + // copied across explicitly or the flags silently become no-ops. + opts.retries = gyp.opts.retries; + opts.retry_delay = gyp.opts.retry_delay; + opts.timeout = gyp.opts.timeout; + opts.node_pre_gyp_retries = gyp.opts.node_pre_gyp_retries; + opts.node_pre_gyp_retry_delay = gyp.opts.node_pre_gyp_retry_delay; + opts.node_pre_gyp_timeout = gyp.opts.node_pre_gyp_timeout; const from = opts.hosted_tarball; const to = opts.module_path; @@ -311,3 +481,18 @@ function install(gyp, argv, callback) { if (process.env.node_pre_gyp_mock_s3) { require('./mock/http')(); } + +/** + * Pure helpers exposed for unit testing the retry decision table without standing up an HTTP round-trip for every + * case. Not part of the public API. + */ +module.exports.__test = { + resolve_retry_opts, + is_retryable_status, + is_retryable_error, + backoff_delay, + DEFAULT_RETRIES, + DEFAULT_RETRY_DELAY, + DEFAULT_TIMEOUT, + MAX_RETRY_DELAY +}; diff --git a/lib/node-pre-gyp.js b/lib/node-pre-gyp.js index bc80dd90..8c05f654 100644 --- a/lib/node-pre-gyp.js +++ b/lib/node-pre-gyp.js @@ -99,7 +99,10 @@ proto.configDefs = { directory: String, // bin proxy: String, // 'install' loglevel: String, // everywhere - acl: String // 'publish' - S3 ACL for published binaries + acl: String, // 'publish' - S3 ACL for published binaries + retries: Number, // 'install' - download retry attempts after the first + retry_delay: Number, // 'install' - base backoff delay in ms + timeout: Number // 'install' - per-attempt request timeout in ms }; /** diff --git a/test/retry.test.js b/test/retry.test.js new file mode 100644 index 00000000..cd072337 --- /dev/null +++ b/test/retry.test.js @@ -0,0 +1,344 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const test = require('tape'); +const nock = require('nock'); +const install = require('../lib/install.js'); + +const { + resolve_retry_opts, + is_retryable_status, + is_retryable_error, + backoff_delay, + DEFAULT_RETRIES, + DEFAULT_RETRY_DELAY, + DEFAULT_TIMEOUT, + MAX_RETRY_DELAY +} = install.__test; + +// Dummy tar.gz data - contains a blank directory +const targz = 'H4sICPr8u1oCA3gudGFyANPTZ6A5MDAwMDc1VQDTZhAaCGA0hGNobGRqZm5uZmxupGBgaGhiZsKgYMpAB1BaXJJYBHRKYk5pcioedeUZqak5+D2J5CkFhlEwCkbBKBjkAAAyG1ofAAYAAA=='; + +const projectRoot = path.join(__dirname, '..'); +const origin = 'https://npg-mock-bucket.s3.us-east-1.amazonaws.com'; + +/** + * Builds the tarball path matcher. Returns a fresh RegExp each call because nock matching against a shared instance + * is stateful and mismatches intermittently. + * + * @returns {RegExp} Matcher for the app1 test tarball path. + */ +function tarballPath() { + return /\/node-pre-gyp\/node-pre-gyp-test-app1\/v0.1.0\/Release\/node-v\d+-\S+.tar.gz/; +} + +/** + * Builds install options for the app1 fixture, pointed at the nock origin. Retry settings are passed through the same + * gyp.opts path the feature uses, so a broken config copy shows up as a slow or failing test rather than a silent pass. + * + * @param {Object} [retryOpts] Retry settings to merge into opts, such as retries and retry_delay. + * @returns {Object} Options object suitable for passing to install(). + */ +function buildOpts(retryOpts = {}) { + const opts = { + opts: Object.assign({ + 'build-from-source': false, + 'update-binary': true, + retries: 3, + retry_delay: 5 + }, retryOpts) + }; + + const appDir = path.join(projectRoot, 'test', 'app1'); + process.chdir(appDir); + opts.package_json = JSON.parse(fs.readFileSync('./package.json')); + opts.package_json.binary.host = origin; + return opts; +} + +test('retries a 500 and succeeds on the second attempt', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).reply(500, 'Internal Server Error') + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ifError(err, 'install should succeed after retrying the 500'); + t.ok(scope.isDone(), 'both the failed and the successful request were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries repeatedly while the host returns 503', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).reply(503, 'Service Unavailable') + .get(tarballPath()).reply(503, 'Service Unavailable') + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts({ retries: 3 }), [], (err) => { + t.ifError(err, 'install should succeed on the third attempt'); + t.ok(scope.isDone(), 'all three requests were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries a 504 gateway timeout', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).reply(504, 'Gateway Time-out') + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ifError(err, 'install should succeed after retrying the 504'); + t.ok(scope.isDone(), 'both requests were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('gives up after exhausting retries and reports the status code', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).reply(500, 'Internal Server Error') + .get(tarballPath()).reply(500, 'Internal Server Error') + .get(tarballPath()).reply(500, 'Internal Server Error'); + + install(buildOpts({ retries: 2 }), [], (err) => { + t.ok(err, 'install should fail once retries are exhausted'); + t.equal(err.statusCode, 500, 'error should carry statusCode for print_fallback_error'); + t.ok(err.message.includes('500'), 'error message should mention the status'); + t.ok(scope.isDone(), 'exactly three attempts were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('reports statusCode on an exhausted 504, for the fallback diagnostic', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).reply(504, 'Gateway Time-out') + .get(tarballPath()).reply(504, 'Gateway Time-out'); + + install(buildOpts({ retries: 1 }), [], (err) => { + t.ok(err, 'install should fail'); + t.equal(err.statusCode, 504, 'error should carry the 504 status code'); + t.ok(scope.isDone(), 'both attempts were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries a 429 rate limit response', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).reply(429, 'Too Many Requests') + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ifError(err, 'install should succeed after retrying the 429'); + t.ok(scope.isDone(), 'both requests were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('does not retry a 404', (t) => { + nock.cleanAll(); + const failing = nock(origin).get(tarballPath()).reply(404, 'Not Found'); + const followUp = nock(origin).get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ok(err, 'install should fail on 404'); + t.equal(err.statusCode, 404, 'error should carry the 404 status code'); + t.ok(failing.isDone(), 'the 404 request was made'); + t.notOk(followUp.isDone(), 'no second request was made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('does not retry a 403, leaving the authenticated path to handle it', (t) => { + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + delete process.env.node_pre_gyp_mock_s3; + + nock.cleanAll(); + const failing = nock(origin).get(tarballPath()).reply(403, 'Forbidden'); + const followUp = nock(origin).get(tarballPath()).reply(403, 'Forbidden'); + + install(buildOpts({ retries: 3 }), [], (err) => { + t.ok(err, 'install should fail without credentials'); + t.equal(err.statusCode, 403, 'error should carry the 403 status code'); + t.ok(err.message.includes('AWS credentials not found'), 'should route to the authenticated path'); + t.ok(failing.isDone(), 'the 403 request was made'); + t.notOk(followUp.isDone(), 'the 403 was not retried'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries a network error carrying a transient code', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).replyWithError({ code: 'ECONNRESET', message: 'read ECONNRESET' }) + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ifError(err, 'install should succeed after retrying the reset connection'); + t.ok(scope.isDone(), 'both requests were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries a bare socket hang up with no error code', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).replyWithError(new Error('socket hang up')) + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ifError(err, 'install should succeed after retrying the hang up'); + t.ok(scope.isDone(), 'both requests were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('does not retry ENOTFOUND', (t) => { + nock.cleanAll(); + const failing = nock(origin) + .get(tarballPath()) + .replyWithError({ code: 'ENOTFOUND', message: 'getaddrinfo ENOTFOUND' }); + const followUp = nock(origin).get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts(), [], (err) => { + t.ok(err, 'install should fail on a DNS miss'); + t.ok(failing.isDone(), 'the failing request was made'); + t.notOk(followUp.isDone(), 'no second request was made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries a request that exceeds the per-attempt timeout', (t) => { + nock.cleanAll(); + const scope = nock(origin) + .get(tarballPath()).delayConnection(200).reply(200, Buffer.from(targz, 'base64')) + .get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts({ timeout: 20 }), [], (err) => { + t.ifError(err, 'install should succeed after the slow attempt times out'); + t.ok(scope.isDone(), 'both requests were made'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('retries: 0 disables retrying', (t) => { + nock.cleanAll(); + const failing = nock(origin).get(tarballPath()).reply(500, 'Internal Server Error'); + const followUp = nock(origin).get(tarballPath()).reply(200, Buffer.from(targz, 'base64')); + + install(buildOpts({ retries: 0 }), [], (err) => { + t.ok(err, 'install should fail immediately'); + t.equal(err.statusCode, 500, 'error should carry the 500 status code'); + t.ok(failing.isDone(), 'the failing request was made'); + t.notOk(followUp.isDone(), 'no retry was attempted'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('is_retryable_status matches the retry decision table', (t) => { + [429, 500, 502, 503, 504, 599].forEach((status) => { + t.ok(is_retryable_status(status), `${status} should be retryable`); + }); + [200, 301, 400, 401, 403, 404, 410, 418].forEach((status) => { + t.notOk(is_retryable_status(status), `${status} should not be retryable`); + }); + t.end(); +}); + +test('is_retryable_error matches the retry decision table', (t) => { + ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE', 'EAI_AGAIN', 'ENETUNREACH', 'EHOSTUNREACH'] + .forEach((code) => { + t.ok(is_retryable_error({ code }), `${code} should be retryable`); + }); + + t.notOk(is_retryable_error({ code: 'ENOTFOUND' }), 'ENOTFOUND should not be retryable'); + t.notOk(is_retryable_error(null), 'a missing error should not be retryable'); + t.notOk(is_retryable_error({ name: 'AbortError' }), 'a deliberate abort should not be retryable'); + t.notOk(is_retryable_error(new Error('something else entirely')), 'an unknown error should not be retryable'); + t.ok(is_retryable_error({ type: 'request-timeout' }), 'a node-fetch request timeout should be retryable'); + t.ok(is_retryable_error({ type: 'body-timeout' }), 'a node-fetch body timeout should be retryable'); + t.ok(is_retryable_error(new Error('socket hang up')), 'a socket hang up should be retryable'); + t.end(); +}); + +test('resolve_retry_opts falls back to defaults', (t) => { + t.deepEqual(resolve_retry_opts({}), { + retries: DEFAULT_RETRIES, + retryDelay: DEFAULT_RETRY_DELAY, + timeout: DEFAULT_TIMEOUT + }, 'an empty opts object yields the defaults'); + + t.deepEqual(resolve_retry_opts({ retries: 5, retry_delay: 10, timeout: 99 }), { + retries: 5, + retryDelay: 10, + timeout: 99 + }, 'explicit options win'); + + t.deepEqual(resolve_retry_opts({ node_pre_gyp_retries: '4', node_pre_gyp_timeout: '50' }), { + retries: 4, + retryDelay: DEFAULT_RETRY_DELAY, + timeout: 50 + }, 'npm-style string config is parsed'); + + t.equal(resolve_retry_opts({ retries: 0 }).retries, 0, 'zero retries is honoured as a disable'); + t.end(); +}); + +test('resolve_retry_opts rejects malformed config rather than producing NaN', (t) => { + const malformed = resolve_retry_opts({ retries: 'lots', retry_delay: 'soon', timeout: 'never' }); + t.equal(malformed.retries, DEFAULT_RETRIES, 'unparseable retries falls back to the default'); + t.equal(malformed.retryDelay, DEFAULT_RETRY_DELAY, 'unparseable delay falls back to the default'); + t.equal(malformed.timeout, DEFAULT_TIMEOUT, 'unparseable timeout falls back to the default, not 0'); + + const negative = resolve_retry_opts({ retries: -1, retry_delay: -1, timeout: -1 }); + t.equal(negative.retries, DEFAULT_RETRIES, 'negative retries falls back to the default'); + t.equal(negative.retryDelay, DEFAULT_RETRY_DELAY, 'negative delay falls back to the default'); + t.equal(negative.timeout, DEFAULT_TIMEOUT, 'negative timeout falls back to the default'); + t.end(); +}); + +test('backoff_delay grows exponentially and stays within its ceiling', (t) => { + for (let attempt = 0; attempt < 6; attempt++) { + const ceiling = Math.min(1000 * Math.pow(2, attempt), MAX_RETRY_DELAY); + const samples = Array.from({ length: 200 }, () => backoff_delay(attempt, 1000)); + const outOfRange = samples.filter((delay) => delay < 0 || delay >= ceiling); + t.equal(outOfRange.length, 0, `attempt ${attempt}: all samples within [0, ${ceiling})`); + t.ok(Math.max(...samples) > ceiling / 4, `attempt ${attempt}: jitter reaches the upper part of the range`); + } + + const capped = Array.from({ length: 200 }, () => backoff_delay(100, 1000)); + t.ok(Math.max(...capped) <= MAX_RETRY_DELAY, 'a large attempt count stays capped'); + t.end(); +});