diff --git a/src/data/chunk-metadata-anchor-source.test.ts b/src/data/chunk-metadata-anchor-source.test.ts index 3a8a7b16b..d14a05884 100644 --- a/src/data/chunk-metadata-anchor-source.test.ts +++ b/src/data/chunk-metadata-anchor-source.test.ts @@ -5,6 +5,8 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ import { strict as assert } from 'node:assert'; +import * as http from 'node:http'; +import { AddressInfo } from 'node:net'; import { afterEach, describe, it } from 'node:test'; import type { AxiosInstance } from 'axios'; @@ -504,4 +506,110 @@ describe('ChunkMetadataAnchorSource', () => { assert.notStrictEqual(result, null); }); }); + + describe('range-GET fallback against a real server', () => { + /** + * A peer whose HEAD carries no chunk headers, so every probe takes the + * range-GET fallback. Records each request's client port so tests can + * tell whether connections were reused. + */ + const startPeer = async (onGet: (res: http.ServerResponse) => void) => { + const clientPorts: number[] = []; + const server = http.createServer((req, res) => { + clientPorts.push(req.socket.remotePort ?? -1); + if (req.method === 'HEAD') { + res.writeHead(200, { 'Content-Length': '0' }); + res.end(); + return; + } + onGet(res); + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', resolve), + ); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + clientPorts, + close: async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }, + }; + }; + + /** A source using its own real axios instance. */ + const makeRealSource = (peerUrl: string) => + new ChunkMetadataAnchorSource({ + log, + peerUrls: [peerUrl], + requestTimeoutMs: 5000, + cacheSize: 32, + cacheTtlMs: 60_000, + fetchTxOffset: async () => matchingChainOffset, + fetchTransaction: async () => ({ data_root: dataRoot }), + }); + + it('does not download the whole body when the peer ignores the range', async () => { + const total = 64 * 1024 * 1024; + const progress = { written: 0, finished: false }; + const peer = await startPeer((res) => { + res.writeHead(200, { + 'Content-Length': String(total), + ...(chunkHeaders() as Record), + }); + const chunk = Buffer.alloc(64 * 1024); + const write = () => { + while (progress.written < total) { + progress.written += chunk.length; + if (!res.write(chunk)) { + res.once('drain', write); + return; + } + } + res.end(); + progress.finished = true; + }; + write(); + }); + try { + const result = await makeRealSource(peer.url).getTxBoundary( + inRangeOffset, + ); + + assert.notEqual(result, null); + assert.strictEqual(result!.id, txId); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(progress.finished, false); + assert.ok( + progress.written < total / 2, + `peer sent ${progress.written} of ${total} bytes`, + ); + } finally { + await peer.close(); + } + }); + + it('keeps reusing the connection when the peer honours the range', async () => { + const peer = await startPeer((res) => { + res.writeHead(206, { + 'Content-Length': '1', + 'Content-Range': 'bytes 0-0/262144', + ...(chunkHeaders() as Record), + }); + res.end(Buffer.from('x')); + }); + try { + const source = makeRealSource(peer.url); + assert.notEqual(await source.getTxBoundary(inRangeOffset), null); + assert.notEqual(await source.getTxBoundary(inRangeOffset + 1n), null); + + // HEAD and GET for each probe, all over one keep-alive connection. + assert.equal(peer.clientPorts.length, 4); + assert.equal(new Set(peer.clientPorts).size, 1); + } finally { + await peer.close(); + } + }); + }); }); diff --git a/src/data/chunk-metadata-anchor-source.ts b/src/data/chunk-metadata-anchor-source.ts index c52125231..1466c697f 100644 --- a/src/data/chunk-metadata-anchor-source.ts +++ b/src/data/chunk-metadata-anchor-source.ts @@ -17,7 +17,7 @@ import { anchorChunkMetadata, } from '../lib/chunk-metadata-anchor.js'; import { createAgentPair } from '../lib/http-agent.js'; -import { normalizeAbortError } from '../lib/http-utils.js'; +import { discardResponseBody, normalizeAbortError } from '../lib/http-utils.js'; // Largest absolute weave offset this source will probe via the // number-typed cache + chain cross-check path. The chain-anchored @@ -301,13 +301,21 @@ export class ChunkMetadataAnchorSource implements TxBoundarySource { } // `bytes=0-0` is the smallest legal range; the server returns a 1- - // byte body which we discard. Headers are the only thing we want. + // byte body which we discard. Headers are the only thing we want. The + // body arrives as a stream so a peer that ignores the range has its + // connection closed instead of its whole response buffered. const getResponse = await this.axiosInstance.get(url, { signal, timeout: this.requestTimeoutMs, headers: { Range: 'bytes=0-0' }, - responseType: 'arraybuffer', + responseType: 'stream', }); + await discardResponseBody(getResponse.data, { + timeoutMs: this.requestTimeoutMs, + }); + // A buffered read rejected when aborted mid-body; keep that behaviour now + // that the body is discarded separately. + signal?.throwIfAborted(); return getResponse.headers as Record; } diff --git a/src/discovery/gateways-root-tx-index.test.ts b/src/discovery/gateways-root-tx-index.test.ts index 93e8123b0..3637e945d 100644 --- a/src/discovery/gateways-root-tx-index.test.ts +++ b/src/discovery/gateways-root-tx-index.test.ts @@ -5,6 +5,8 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ import { strict as assert } from 'node:assert'; +import * as http from 'node:http'; +import { AddressInfo } from 'node:net'; import { afterEach, describe, it, mock } from 'node:test'; import { LRUCache } from 'lru-cache'; import { GatewaysRootTxIndex } from './gateways-root-tx-index.js'; @@ -824,4 +826,154 @@ describe('GatewaysRootTxIndex', () => { ); }); }); + + describe('range-GET fallback against a real server', () => { + const ROOT_HEADERS = { + 'X-AR-IO-Root-Transaction-Id': 'root-tx-456', + 'X-AR-IO-Root-Data-Item-Offset': '1000', + 'X-AR-IO-Root-Data-Offset': '1500', + }; + + /** + * A /raw server that rejects HEAD, as some peers behind CDNs do, so every + * lookup takes the range-GET fallback. Records each request's client port + * so tests can tell whether connections were reused. + */ + const startServer = async (onGet: (res: http.ServerResponse) => void) => { + const clientPorts: number[] = []; + const server = http.createServer((req, res) => { + clientPorts.push(req.socket.remotePort ?? -1); + if (req.method === 'HEAD') { + res.writeHead(405, { 'Content-Length': '0' }); + res.end(); + return; + } + onGet(res); + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', resolve), + ); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + clientPorts, + close: async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }, + }; + }; + + /** + * Streams `total` bytes with backpressure and reports how much the client + * actually accepted before hanging up. + */ + const streamLargeBody = ( + res: http.ServerResponse, + status: number, + total: number, + headers: Record, + ) => { + const progress = { written: 0, finished: false }; + res.writeHead(status, { 'Content-Length': String(total), ...headers }); + const chunk = Buffer.alloc(64 * 1024); + const write = () => { + while (progress.written < total) { + progress.written += chunk.length; + if (!res.write(chunk)) { + res.once('drain', write); + return; + } + } + res.end(); + progress.finished = true; + }; + write(); + return progress; + }; + + const makeIndex = (url: string) => { + const index = new GatewaysRootTxIndex({ + log, + trustedGatewaysUrls: { [url]: 1 }, + requestTimeoutMs: 5000, + rateLimitBurstSize: 1000, + rateLimitTokensPerInterval: 1000, + rateLimitInterval: 'second', + }); + for (const [, limiter] of (index as any)['limiters']) { + limiter.content = limiter.bucketSize; + } + return index; + }; + + const TOTAL = 64 * 1024 * 1024; + + it('does not download the whole item when the peer ignores the range', async () => { + let progress = { written: 0, finished: false }; + const server = await startServer((res) => { + progress = streamLargeBody(res, 200, TOTAL, ROOT_HEADERS); + }); + try { + const result = await makeIndex(server.url).getRootTx('item-a'); + + assert(result !== undefined); + assert.equal(result.rootTxId, 'root-tx-456'); + // A 200 carries the whole payload, so Content-Length is its size. + assert.equal(result.dataSize, TOTAL); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(progress.finished, false); + assert.ok( + progress.written < TOTAL / 2, + `peer sent ${progress.written} of ${TOTAL} bytes`, + ); + } finally { + await server.close(); + } + }); + + it('does not download a large error body either', async () => { + let progress = { written: 0, finished: false }; + const server = await startServer((res) => { + progress = streamLargeBody(res, 500, TOTAL, {}); + }); + try { + const result = await makeIndex(server.url).getRootTx('item-a'); + + assert.equal(result, undefined); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(progress.finished, false); + assert.ok( + progress.written < TOTAL / 2, + `peer sent ${progress.written} of ${TOTAL} bytes`, + ); + } finally { + await server.close(); + } + }); + + it('keeps reusing the connection when the peer honours the range', async () => { + const server = await startServer((res) => { + res.writeHead(206, { + 'Content-Length': '1', + 'Content-Range': 'bytes 0-0/5000', + ...ROOT_HEADERS, + }); + res.end(Buffer.from('x')); + }); + try { + const index = makeIndex(server.url); + const first = await index.getRootTx('item-a'); + const second = await index.getRootTx('item-b'); + + assert.equal(first?.dataSize, 5000); + assert.equal(second?.dataSize, 5000); + // HEAD and GET for each lookup, all over one keep-alive connection. + assert.equal(server.clientPorts.length, 4); + assert.equal(new Set(server.clientPorts).size, 1); + } finally { + await server.close(); + } + }); + }); }); diff --git a/src/discovery/gateways-root-tx-index.ts b/src/discovery/gateways-root-tx-index.ts index ddf1d41ec..0ff2bed15 100644 --- a/src/discovery/gateways-root-tx-index.ts +++ b/src/discovery/gateways-root-tx-index.ts @@ -10,7 +10,11 @@ import { LRUCache } from 'lru-cache'; import { TokenBucket } from 'limiter'; import { DataItemRootIndex } from '../types.js'; import { shuffleArray } from '../lib/random.js'; -import { parseContentRange, parseNonNegativeInt } from '../lib/http-utils.js'; +import { + discardResponseBody, + parseContentRange, + parseNonNegativeInt, +} from '../lib/http-utils.js'; import { createAgentPair } from '../lib/http-agent.js'; import * as config from '../config.js'; import * as metrics from '../metrics.js'; @@ -46,6 +50,7 @@ export class GatewaysRootTxIndex implements DataItemRootIndex { private readonly axiosInstance: AxiosInstance; private readonly cache?: LRUCache; private readonly limiters: Map; + private readonly requestTimeoutMs: number; constructor({ log, @@ -66,6 +71,7 @@ export class GatewaysRootTxIndex implements DataItemRootIndex { }) { this.log = log.child({ class: this.constructor.name }); this.cache = cache; + this.requestTimeoutMs = requestTimeoutMs; if (Object.keys(trustedGatewaysUrls).length === 0) { throw new Error('At least one gateway URL must be provided'); @@ -296,7 +302,9 @@ export class GatewaysRootTxIndex implements DataItemRootIndex { * those behind CDNs or proxies — don't support HEAD on this route even * though the upstream gateway would. `bytes=0-0` is the smallest legal * range; the server returns 1 byte of body we discard, and the headers - * are what we want. + * are what we want. The body is received as a stream and discarded by + * {@link discardResponseBody}: a peer that ignores the range and sends the + * whole item has its connection closed instead of the item being buffered. * * 404 is treated as a definitive "item doesn't exist on this peer" and * propagated to the caller — falling back to GET would just hit the @@ -316,10 +324,22 @@ export class GatewaysRootTxIndex implements DataItemRootIndex { throw err; } // Network error, 405 Method Not Allowed, 5xx, etc. — try GET. - return this.axiosInstance.get(url, { - headers: { Range: 'bytes=0-0' }, - responseType: 'arraybuffer', - }); + try { + const response = await this.axiosInstance.get(url, { + headers: { Range: 'bytes=0-0' }, + responseType: 'stream', + }); + await discardResponseBody(response.data, { + timeoutMs: this.requestTimeoutMs, + }); + return response; + } catch (getErr: any) { + // An error status still carries an unread body stream. + await discardResponseBody(getErr?.response?.data, { + timeoutMs: this.requestTimeoutMs, + }); + throw getErr; + } } } } diff --git a/src/lib/http-utils.test.ts b/src/lib/http-utils.test.ts index a59a64bfb..259b43f14 100644 --- a/src/lib/http-utils.test.ts +++ b/src/lib/http-utils.test.ts @@ -5,12 +5,14 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ import { strict as assert } from 'node:assert'; +import { Readable } from 'node:stream'; import { describe, it, mock } from 'node:test'; import { buildMultipartResponseParts, buildRangeHeader, calculateMultipartSize, calculateRangeResponseSize, + discardResponseBody, generateBoundary, handleIfNoneMatch, normalizeAbortError, @@ -728,4 +730,93 @@ describe('http-utils', () => { assert.equal(normalizeAbortError(undefined), undefined); }); }); + + describe('discardResponseBody', () => { + /** A stream that yields `chunks` chunks of `chunkSize` bytes, then ends. */ + const chunkedStream = (chunkSize: number, chunks: number) => { + let sent = 0; + return { + stream: new Readable({ + read() { + if (sent >= chunks) { + this.push(null); + return; + } + sent++; + this.push(Buffer.alloc(chunkSize)); + }, + }), + chunksSent: () => sent, + }; + }; + + it('reads a small body to the end without destroying it', async () => { + const { stream } = chunkedStream(1, 1); + + await discardResponseBody(stream, { timeoutMs: 1000 }); + + // Ending normally is what lets a keep-alive socket be reused. + assert.equal(stream.readableEnded, true); + }); + + it('destroys a body larger than the limit instead of reading it all', async () => { + const { stream, chunksSent } = chunkedStream(16 * 1024, 1000); // ~16 MiB + + await discardResponseBody(stream, { + maxBytes: 64 * 1024, + timeoutMs: 1000, + }); + + assert.equal(stream.destroyed, true); + assert.equal(stream.readableEnded, false); + assert.ok(chunksSent() < 20, `read ${chunksSent()} chunks`); + }); + + it('destroys a body that is still arriving after the timeout', async () => { + const stream = new Readable({ read() {} }); // never ends + stream.push(Buffer.alloc(1)); + + await discardResponseBody(stream, { timeoutMs: 50 }); + + assert.equal(stream.destroyed, true); + }); + + it('resolves when the stream fails, without an unhandled error', async () => { + const stream = new Readable({ read() {} }); + const done = discardResponseBody(stream, { timeoutMs: 1000 }); + stream.destroy(new Error('socket hang up')); + + await done; + + // A later error has a listener, so it cannot crash the process. + assert.ok(stream.listenerCount('error') > 0); + }); + + it('handles an error still pending on an already destroyed stream', async () => { + // destroy(error) sets `destroyed` immediately but emits 'error' on a + // later tick. Without a listener that error would be uncaught. + const stream = new Readable({ read() {} }); + stream.destroy(new Error('late socket error')); + assert.equal(stream.destroyed, true); + + await discardResponseBody(stream, { timeoutMs: 1000 }); + await new Promise((resolve) => setImmediate(resolve)); + + assert.ok(stream.listenerCount('error') > 0); + }); + + it('ignores values that are not readable streams', async () => { + await discardResponseBody(undefined, { timeoutMs: 10 }); + await discardResponseBody(Buffer.alloc(4), { timeoutMs: 10 }); + await discardResponseBody('body', { timeoutMs: 10 }); + }); + + it('returns at once for a stream that has already ended', async () => { + const stream = Readable.from([]); + stream.resume(); + await new Promise((resolve) => stream.once('end', resolve)); + + await discardResponseBody(stream, { timeoutMs: 10_000 }); + }); + }); }); diff --git a/src/lib/http-utils.ts b/src/lib/http-utils.ts index 96f21aaa4..a557173d9 100644 --- a/src/lib/http-utils.ts +++ b/src/lib/http-utils.ts @@ -5,6 +5,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ import { randomBytes } from 'node:crypto'; +import { Readable } from 'node:stream'; import rangeParser from 'range-parser'; import { Request, Response } from 'express'; @@ -361,3 +362,75 @@ export function normalizeAbortError(error: any): any { } return error; } + +/** Largest response body {@link discardResponseBody} reads to the end. */ +export const DISCARDED_BODY_MAX_BYTES = 64 * 1024; + +/** + * Discards a response body that was requested as a stream only for its + * headers, such as the `Range: bytes=0-0` GET used when a peer rejects HEAD. + * + * A body of up to `maxBytes` is read to the end, so a keep-alive socket goes + * back to its pool exactly as it would after a buffered read. A larger body (a + * peer that ignored the range and is sending the whole item), a body still + * arriving after `timeoutMs`, or a stream that fails is destroyed instead, + * which closes the connection rather than downloading the rest. + * + * Never rejects. Values that aren't readable streams are ignored. + */ +export async function discardResponseBody( + body: unknown, + { + maxBytes = DISCARDED_BODY_MAX_BYTES, + timeoutMs, + }: { maxBytes?: number; timeoutMs: number }, +): Promise { + if (!(body instanceof Readable)) { + return; + } + const stream: Readable = body; + + // A stream failing after we stop listening must not raise an unhandled + // 'error' event. This goes before the checks below: `destroy(error)` marks + // the stream destroyed at once but emits 'error' on a later tick, so a + // stream can already be destroyed with its error still pending. + stream.on('error', () => {}); + if (stream.destroyed || stream.readableEnded) { + return; + } + + await new Promise((resolve) => { + let received = 0; + // Created before any listener, so every callback below can clear it. + const timer = setTimeout(() => finish(true), timeoutMs); + + function finish(destroy: boolean): void { + clearTimeout(timer); + stream.off('data', onData); + stream.off('end', onDone); + stream.off('close', onDone); + stream.off('error', onError); + if (destroy && !stream.destroyed) { + stream.destroy(); + } + resolve(); + } + function onData(chunk: Buffer): void { + received += chunk.length; + if (received > maxBytes) { + finish(true); + } + } + function onDone(): void { + finish(false); + } + function onError(): void { + finish(true); + } + + stream.on('data', onData); + stream.once('end', onDone); + stream.once('close', onDone); + stream.once('error', onError); + }); +}