diff --git a/packages/helpers/simple/browser-budget.json b/packages/helpers/simple/browser-budget.json index 249bc9ee9..3e44031a0 100644 --- a/packages/helpers/simple/browser-budget.json +++ b/packages/helpers/simple/browser-budget.json @@ -25,9 +25,9 @@ ], "maximumBytes": { "vite": { - "raw": 750000, + "raw": 755000, "gzip": 185000, - "brotli": 150000 + "brotli": 151500 }, "esbuild": { "raw": 585000, diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index e78e6e082..591873c13 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -270,9 +270,26 @@ All notable changes to this project will be documented in this file. The format - Add `LookupResolver.queryDetailed()` and per-outcome host-settlement counts so security-sensitive callers can distinguish authoritative empty answers from partial availability. +- Add bounded LookupResolver discovery: later SLAP tracker advertisements can + join an active query, hosts are scheduled fairly under concurrency and byte + limits, and HTTP bodies are read incrementally. Raw `query$` output remains + unverified; `onEvidence` is the C02 intake seam. Existing 2s host / 5s + tracker delays, reputation/backoff, freeform answers, and CORS/public + lookup request headers are unchanged. ### Changed +- LookupResolver host cache no longer lets a tighter-limit discovery satisfy a + later larger query, and `query()` still throws the historical no-competent-hosts + error when a deadline expires before any host is admitted. +- `LookupResolver.query()` and `queryDetailed()` now reject with an `AbortError` + when the caller's `options.signal` cancels the attempt, at any host count, + instead of flattening a cancelled run into an empty output list. `query$()` + still reports the cancellation as a `terminalReason: 'cancelled'` snapshot. +- A lookup that exhausts a client resource budget during SLAP discovery, before + any host is admitted, now throws `LookupResourceLimitError` naming the limit + instead of the historical no-competent-hosts error. That message is reserved + for a deadline or a settled attempt that genuinely found no host. - Batch BEEF mutation bookkeeping and reuse compound Merkle intermediate hashes. The optional asynchronous P2PKH backend now forwards its already validated compressed public key directly into the unlocking script. Existing BEEF @@ -388,6 +405,12 @@ All notable changes to this project will be documented in this file. The format ### Security +- `HTTPSOverlayLookupFacilitator` now issues lookup and SLAP tracker discovery + requests with `redirect: 'error'`. A SLAP-advertised host can no longer + redirect the serialized lookup body to an origin that the advertised-host + scheme and credential checks never saw, such as `http:`, loopback, or + link-local. A redirected response is recorded as an ordinary availability + failure for the advertised host. - Treat cryptographic verification as successful only when it returns an affirmative result: `GlobalKVStore` rejects forged controller-signed overlay values, and `IdentityClient` refuses to publish signature-invalid identity diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 6705fcce5..822515491 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 165000 }, "esbuild": { - "raw": 590000, + "raw": 600000, "gzip": 180000, "brotli": 150000 } @@ -30,7 +30,7 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 590000, + "raw": 594000, "gzip": 172000, "brotli": 145000 } diff --git a/packages/sdk/docs/reference/overlay-tools.md b/packages/sdk/docs/reference/overlay-tools.md index 8c5492e27..c1118dddb 100644 --- a/packages/sdk/docs/reference/overlay-tools.md +++ b/packages/sdk/docs/reference/overlay-tools.md @@ -6,13 +6,15 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | | --- | --- | -| [AdmittanceInstructions](#interface-admittanceinstructions) | [OverlayBroadcastFacilitator](#interface-overlaybroadcastfacilitator) | -| [LookupAnswerProgress](#interface-lookupanswerprogress) | [OverlayLookupFacilitator](#interface-overlaylookupfacilitator) | -| [LookupFreeformAnswer](#interface-lookupfreeformanswer) | [RankedHost](#interface-rankedhost) | +| [AdmittanceInstructions](#interface-admittanceinstructions) | [LookupResolverConfig](#interface-lookupresolverconfig) | +| [LookupAnswerProgress](#interface-lookupanswerprogress) | [LookupResponseReaderOptions](#interface-lookupresponsereaderoptions) | +| [LookupDiscoveryUpdate](#interface-lookupdiscoveryupdate) | [OverlayBroadcastFacilitator](#interface-overlaybroadcastfacilitator) | +| [LookupFreeformAnswer](#interface-lookupfreeformanswer) | [OverlayLookupFacilitator](#interface-overlaylookupfacilitator) | +| [LookupLimits](#interface-lookuplimits) | [RankedHost](#interface-rankedhost) | | [LookupQueryOptions](#interface-lookupqueryoptions) | [SHIPBroadcasterConfig](#interface-shipbroadcasterconfig) | | [LookupQuestion](#interface-lookupquestion) | [TaggedBEEF](#interface-taggedbeef) | -| [LookupResolution](#interface-lookupresolution) | [UnreachableHostInfo](#interface-unreachablehostinfo) | -| [LookupResolverConfig](#interface-lookupresolverconfig) | | +| [LookupRequestOptions](#interface-lookuprequestoptions) | [UnreachableHostInfo](#interface-unreachablehostinfo) | +| [LookupResolution](#interface-lookupresolution) | | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -62,6 +64,17 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts export interface LookupAnswerProgress { + discoveryComplete?: boolean; + terminalReason?: "settled" | "deadline" | "cancelled" | "resource-limit"; + discoveredHosts?: number; + skippedHosts?: number; + receivedBytes?: number; + retainedBytes?: number; + evidenceBytes?: number; + trackersTotal?: number; + trackersCompleted?: number; + trackersFailed?: number; + limitsHit?: string[]; type: "output-list"; outputs: Array<{ beef: number[]; @@ -98,6 +111,14 @@ Correlation id used for privacy-safe distributed diagnostics. correlationId?: string ``` +#### Property discoveryComplete + +Transport coverage only, never cryptographic validity or global absence. + +```ts +discoveryComplete?: boolean +``` + #### Property emptyHosts Successful hosts whose output list was empty. @@ -106,6 +127,14 @@ Successful hosts whose output list was empty. emptyHosts: number ``` +#### Property evidenceBytes + +Receipt-copy octets handed to onEvidence, independently bounded. + +```ts +evidenceBytes?: number +``` + #### Property failedHosts Hosts that failed due to availability, timeout, or malformed responses. @@ -146,6 +175,14 @@ Hosts that rejected this query semantically (for example, HTTP 400). rejectedHosts: number ``` +#### Property retainedBytes + +Retained decoded BEEF/context octets; JavaScript arrays have additional heap overhead. + +```ts +retainedBytes?: number +``` + #### Property successfulHosts Hosts that returned a structurally valid output-list response. @@ -164,6 +201,24 @@ txIds: string[] Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: LookupDiscoveryUpdate + +```ts +export interface LookupDiscoveryUpdate { + sources: Map; + trackersTotal: number; + trackersCompleted: number; + trackersFailed: number; + skippedHosts: number; + receivedBytes: number; + limitsHit: Set; + done: boolean; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Interface: LookupFreeformAnswer @@ -178,16 +233,41 @@ export interface LookupFreeformAnswer { Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: LookupLimits + +Operational client limits, not BEEF validity or service authority rules. + +```ts +export interface LookupLimits { + maxHosts: number; + maxHostsPerTracker: number; + maxTrackers: number; + hostConcurrency: number; + trackerConcurrency: number; + maxResponseBytes: number; + maxTotalBytes: number; + maxOutputs: number; + maxEvidenceOutputs: number; + maxEvidenceBytes: number; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Interface: LookupQueryOptions ```ts export interface LookupQueryOptions { - onEvidence?: (event: LookupEvidenceEvent) => void | Promise; + signal?: AbortSignal; evidenceLimits?: { maxOutputs?: number; maxBytes?: number; }; + deadlineMs?: number; + limits?: Partial; + onEvidence?: (event: LookupEvidenceEvent) => void | Promise; graceMs?: number; softTimeoutMs?: number; onUnreachableHost?: (info: UnreachableHostInfo) => void | Promise; @@ -198,7 +278,7 @@ export interface LookupQueryOptions { } ``` -See also: [LookupEvidenceEvent](./overlay-tools.md#type-lookupevidenceevent), [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) +See also: [LookupEvidenceEvent](./overlay-tools.md#type-lookupevidenceevent), [LookupLimits](./overlay-tools.md#interface-lookuplimits), [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) #### Property correlationId @@ -208,11 +288,22 @@ Correlates resolver and downstream wallet telemetry without logging the query pa correlationId?: string ``` +#### Property deadlineMs + +Whole attempt budget including discovery and queued hosts. Default 10000 ms. + +```ts +deadlineMs?: number +``` + #### Property evidenceLimits Callback intake budget, independent of legacy aggregation. Defaults to 512 outputs / 16 MiB of BEEF and context bytes. Values must be positive safe integers. Coordinate these with a downstream verifier's admission limits. +Precedence when both this and `limits.maxEvidenceOutputs`/`maxEvidenceBytes` +are supplied for the same call: `evidenceLimits` wins, then `limits`, then +the resolver's configured limits, then the library defaults. ```ts evidenceLimits?: { @@ -240,13 +331,25 @@ code. `waitForAllHosts` takes precedence when both are supplied. holdForUnknownHosts?: boolean ``` +#### Property limits + +Per-query operational resource limits (discovery, transport and queueing +bounds). `limits.maxEvidenceOutputs`/`maxEvidenceBytes` also set the +evidence intake budget, but the `evidenceLimits` shorthand above takes +precedence over these two fields when both are supplied. + +```ts +limits?: Partial +``` +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + #### Property onEvidence Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue promptly; callback completion is not awaited and failures are isolated. Intake stops at the configured evidenceLimits, reporting one limit event. -No callbacks occur after the query iterator closes. Legacy answers, host -scheduling, timeout and reputation behavior are unchanged. +No callbacks occur after the query iterator closes. Raw `query$` snapshots +remain unverified transport aggregates, not cryptographic proof. ```ts onEvidence?: (event: LookupEvidenceEvent) => void | Promise @@ -265,6 +368,18 @@ onUnreachableHost?: (info: UnreachableHostInfo) => void | Promise ``` See also: [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) +#### Property signal + +Abort this query without cancelling discovery still owned by another query. +`query()` and `queryDetailed()` reject with an `AbortError` once this +signal fires: a cancelled attempt never answered the question, so it is +never reported as an empty output list. `query$()` keeps emitting its +terminal snapshot with `terminalReason: 'cancelled'` instead. + +```ts +signal?: AbortSignal +``` + #### Property softTimeoutMs Soft timeout (ms). When set: @@ -331,6 +446,21 @@ service: string Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: LookupRequestOptions + +Optional bounded transport settings; older custom facilitators may ignore these. + +```ts +export interface LookupRequestOptions { + maxResponseBytes?: number; + maxOutputs?: number; + consumeBytes?: (bytes: number) => void; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Interface: LookupResolution @@ -354,6 +484,7 @@ Configuration options for the Lookup resolver. ```ts export interface LookupResolverConfig { + limits?: Partial; networkPreset?: LookupNetworkPreset; facilitator?: OverlayLookupFacilitator; slapTrackers?: string[]; @@ -368,7 +499,7 @@ export interface LookupResolverConfig { } ``` -See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits), [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) #### Property additionalHosts @@ -403,6 +534,15 @@ Map of lookup service names to arrays of hosts to use in place of resolving via hostOverrides?: Record ``` +#### Property limits + +Defaults for the bounded discovery, scheduler and receipt intake. + +```ts +limits?: Partial +``` +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + #### Property networkPreset The network preset to use, unless other options override it. @@ -445,6 +585,45 @@ telemetry?: TelemetryConfig Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: LookupResponseReaderOptions + +Options controlling a bounded lookup response read. + +```ts +export interface LookupResponseReaderOptions { + signal?: AbortSignal; + maxResponseBytes: number; + consumeBytes?: (bytes: number) => void; +} +``` + +#### Property consumeBytes + +Charges accepted bytes to the caller's aggregate response budget. + +```ts +consumeBytes?: (bytes: number) => void +``` + +#### Property maxResponseBytes + +Maximum number of response bytes to retain. + +```ts +maxResponseBytes: number +``` + +#### Property signal + +Cancels a pending stream read when the lookup request is aborted. + +```ts +signal?: AbortSignal +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Interface: OverlayBroadcastFacilitator @@ -467,20 +646,20 @@ Facilitates lookups to URLs that return answers. ```ts export interface OverlayLookupFacilitator { - lookup: (url: string, question: LookupQuestion, timeout?: number) => Promise; + lookup: (url: string, question: LookupQuestion, timeout?: number, signal?: AbortSignal, options?: LookupRequestOptions) => Promise; } ``` -See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupRequestOptions](./overlay-tools.md#interface-lookuprequestoptions) #### Property lookup Returns a lookup answer for a lookup question ```ts -lookup: (url: string, question: LookupQuestion, timeout?: number) => Promise +lookup: (url: string, question: LookupQuestion, timeout?: number, signal?: AbortSignal, options?: LookupRequestOptions) => Promise ``` -See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupRequestOptions](./overlay-tools.md#interface-lookuprequestoptions) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -644,8 +823,11 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | [HTTPSOverlayBroadcastFacilitator](#class-httpsoverlaybroadcastfacilitator) | | [HTTPSOverlayLookupFacilitator](#class-httpsoverlaylookupfacilitator) | | [HostReputationTracker](#class-hostreputationtracker) | +| [LookupDiscovery](#class-lookupdiscovery) | | [LookupHTTPError](#class-lookuphttperror) | +| [LookupHostQueue](#class-lookuphostqueue) | | [LookupResolver](#class-lookupresolver) | +| [LookupResourceLimitError](#class-lookupresourcelimiterror) | | [OverlayAdminTokenTemplate](#class-overlayadmintokentemplate) | | [TopicBroadcaster](#class-topicbroadcaster) | @@ -676,11 +858,11 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { fetchClient: typeof fetch; allowHTTP: boolean; constructor(httpClient = defaultFetch, allowHTTP: boolean = false) - async lookup(url: string, question: LookupQuestion, timeout: number = 2000): Promise + async lookup(url: string, question: LookupQuestion, timeout: number = 2000, signal?: AbortSignal, options?: LookupRequestOptions): Promise } ``` -See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupRequestOptions](./overlay-tools.md#interface-lookuprequestoptions), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -711,6 +893,24 @@ flush(): void Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Class: LookupDiscovery + +One bounded refresh shared only by subscribers of the same resolver/configuration. + +```ts +export class LookupDiscovery { + readonly controller = new AbortController(); + readonly state: LookupDiscoveryUpdate; + constructor(private readonly trackers: string[], private readonly limits: LookupLimits, private readonly lookup: (tracker: string, signal: AbortSignal, consume: (bytes: number) => void) => Promise, private readonly finish: (state: LookupDiscoveryUpdate, abandoned: boolean) => void) + subscribe(listener: (state: LookupDiscoveryUpdate) => void): () => void +} +``` + +See also: [LookupDiscoveryUpdate](./overlay-tools.md#interface-lookupdiscoveryupdate), [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Class: LookupHTTPError @@ -728,6 +928,23 @@ See also: [LookupHTTPErrorKind](./overlay-tools.md#type-lookuphttperrorkind) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Class: LookupHostQueue + +A bounded FIFO within each source, round-robin between sources. + +```ts +export class LookupHostQueue { + readonly done = new Promise(resolve => { this.resolveDone = resolve; }); + constructor(private readonly maxHosts: number, private readonly concurrency: number, private readonly run: (host: string) => Promise, private readonly skipped: (count: number, limited: boolean) => void) + add(source: string, hosts: string[]): void + finishSources(): void + cancel(): void +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Class: LookupResolver @@ -738,7 +955,7 @@ export default class LookupResolver { constructor(config: LookupResolverConfig = {}) async query(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise - async *query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable + query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable } ``` @@ -752,17 +969,39 @@ Optional `options.graceMs` overrides the per-call grace window (default 80 ms). Optional `options.softTimeoutMs` resolves the query early with whatever has arrived once any host has answered (or with an empty result if no host has answered by `softTimeoutMs`). +Throws an `AbortError` when `options.signal` aborted the attempt, so a +cancelled lookup is never mistaken for an authoritative empty answer. + ```ts async query(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise ``` See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) +#### Method query$ + +Cumulative unverified results. Discovery remains subscribed while trackers +settle; each new host enters the bounded queue immediately. Caller abort, +deadline and iterator close release this query's ownership. + +```ts +query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable +``` +See also: [LookupAnswerProgress](./overlay-tools.md#interface-lookupanswerprogress), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) + #### Method queryDetailed Performs a lookup and returns both its answer and the host settlement evidence required by security-sensitive consumers to distinguish an authoritative empty result from an availability failure. +Throws an `AbortError` when `options.signal` aborted the attempt, rather +than returning a resolution whose empty answer would have to be +re-qualified against `progress.terminalReason`. When a client resource +budget was exhausted during SLAP discovery, before any host could be +admitted, it throws `LookupResourceLimitError` naming that limit; the +historical no-competent-hosts error is reserved for a deadline or a +settled attempt that genuinely found no host. + ```ts async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise ``` @@ -770,6 +1009,17 @@ See also: [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Class: LookupResourceLimitError + +```ts +export class LookupResourceLimitError extends Error { + constructor(readonly limit: string) +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Class: OverlayAdminTokenTemplate @@ -930,6 +1180,66 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ## Functions +| | +| --- | +| [lookupAbortError](#function-lookupaborterror) | +| [lookupLimits](#function-lookuplimits) | +| [normalizeLookupHost](#function-normalizelookuphost) | +| [readLookupResponseBytes](#function-readlookupresponsebytes) | +| [withDoubleSpendRetry](#function-withdoublespendretry) | +| [withLookupAbort](#function-withlookupabort) | + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: lookupAbortError + +```ts +export function lookupAbortError(): Error +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Function: lookupLimits + +```ts +export function lookupLimits(...overrides: Array | undefined>): LookupLimits +``` + +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Function: normalizeLookupHost + +Preserve distinct paths and ports; remove only a final slash and URL fragments. + +```ts +export function normalizeLookupHost(host: string, allowParameters: boolean = false): string | null +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Function: readLookupResponseBytes + +Reads a lookup response incrementally while enforcing a per-response bound. + +This deliberately does not use Response.text(), json(), or arrayBuffer(), +because those APIs buffer the complete body before a limit can be enforced. + +```ts +export async function readLookupResponseBytes(response: Response, options: LookupResponseReaderOptions): Promise +``` + +See also: [LookupResponseReaderOptions](./overlay-tools.md#interface-lookupresponsereaderoptions) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- ### Function: withDoubleSpendRetry Executes an operation with automatic retry logic for double-spend errors. @@ -961,6 +1271,17 @@ If max retries exceeded or non-double-spend error occurs Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Function: withLookupAbort + +A non-cooperative transport cannot retain a cancelled waiter. + +```ts +export async function withLookupAbort(work: Promise, signal?: AbortSignal): Promise +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ## Types @@ -1093,6 +1414,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | --- | +| [DEFAULT_LOOKUP_LIMITS](#variable-default_lookup_limits) | | [DEFAULT_SLAP_TRACKERS](#variable-default_slap_trackers) | | [DEFAULT_TESTNET_SLAP_TRACKERS](#variable-default_testnet_slap_trackers) | | [DEFAULT_TTN_SLAP_TRACKERS](#variable-default_ttn_slap_trackers) | @@ -1102,6 +1424,28 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- +### Variable: DEFAULT_LOOKUP_LIMITS + +```ts +DEFAULT_LOOKUP_LIMITS: Readonly = Object.freeze({ + maxHosts: 256, + maxHostsPerTracker: 64, + maxTrackers: 16, + hostConcurrency: 8, + trackerConcurrency: 4, + maxResponseBytes: 32 * 1024 * 1024, + maxTotalBytes: 64 * 1024 * 1024, + maxOutputs: 4096, + maxEvidenceOutputs: 512, + maxEvidenceBytes: 16 * 1024 * 1024 +}) +``` + +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- ### Variable: DEFAULT_SLAP_TRACKERS ```ts diff --git a/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts b/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts index f9f0e1881..9cd8c4511 100644 --- a/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts +++ b/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts @@ -2,11 +2,13 @@ import { HTTPSOverlayLookupFacilitator } from '../overlay-tools/LookupResolver' describe('overlay lookup BRC-100 byte compatibility', () => { it('keeps typed query bytes portable across the JSON request boundary', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'output-list', outputs: [] }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ type: 'output-list', outputs: [] }), { + headers: { 'content-type': 'application/json' } + }) + ) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await facilitator.lookup('http://host', { @@ -21,14 +23,17 @@ describe('overlay lookup BRC-100 byte compatibility', () => { }) it('recovers historical numeric-key BEEF from a JSON response', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ - type: 'output-list', - outputs: [{ beef: { 0: 1, 1: 2, 2: 255 }, outputIndex: 0 }] - }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ + type: 'output-list', + outputs: [{ beef: { 0: 1, 1: 2, 2: 255 }, outputIndex: 0 }] + }), + { headers: { 'content-type': 'application/json' } } + ) + ) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await expect( @@ -41,11 +46,13 @@ describe('overlay lookup BRC-100 byte compatibility', () => { it('preserves byte-like objects inside freeform lookup results', async () => { const result = { data: { 0: 1, 1: 2 }, tx: {}, payload: { 0: 3 } } - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'freeform', result }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ type: 'freeform', result }), { + headers: { 'content-type': 'application/json' } + }) + ) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await expect( diff --git a/packages/sdk/src/overlay-tools/LookupDiscovery.ts b/packages/sdk/src/overlay-tools/LookupDiscovery.ts new file mode 100644 index 000000000..8a97eb2ac --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupDiscovery.ts @@ -0,0 +1,120 @@ +import { LookupLimits, LookupResourceLimitError, normalizeLookupHost } from './LookupResources.js' + +export interface LookupDiscoveryUpdate { + sources: Map + trackersTotal: number + trackersCompleted: number + trackersFailed: number + skippedHosts: number + receivedBytes: number + limitsHit: Set + done: boolean +} + +/** One bounded refresh shared only by subscribers of the same resolver/configuration. */ +export class LookupDiscovery { + readonly controller = new AbortController() + readonly state: LookupDiscoveryUpdate + private readonly listeners = new Set<(state: LookupDiscoveryUpdate) => void>() + private started = false + private abandoned = false + private readonly consume = (bytes: number): void => { + if (this.abandoned) throw new LookupResourceLimitError('abandoned') + if (bytes > this.limits.maxTotalBytes - this.state.receivedBytes) { + this.state.limitsHit.add('maxTotalBytes') + throw new LookupResourceLimitError('maxTotalBytes') + } + this.state.receivedBytes += bytes + this.emit() + } + + constructor( + private readonly trackers: string[], + private readonly limits: LookupLimits, + private readonly lookup: (tracker: string, signal: AbortSignal, consume: (bytes: number) => void) => Promise, + private readonly finish: (state: LookupDiscoveryUpdate, abandoned: boolean) => void + ) { + this.state = { + sources: new Map(), trackersTotal: trackers.length, trackersCompleted: 0, + trackersFailed: 0, skippedHosts: 0, receivedBytes: 0, limitsHit: new Set(), done: false + } + } + + subscribe(listener: (state: LookupDiscoveryUpdate) => void): () => void { + this.listeners.add(listener) + listener(this.state) + if (!this.started) { this.started = true; void this.run() } + return () => { + this.listeners.delete(listener) + if (this.listeners.size === 0 && !this.state.done) { + this.abandoned = true + this.controller.abort() + this.finish(this.state, true) + } + } + } + + private emit(): void { + if (this.abandoned) return + for (const listener of this.listeners) listener(this.state) + } + + private collectTrackerHosts(candidates: string[], share: number): string[] { + const hosts = new Set() + for (const candidate of candidates) { + const host = normalizeLookupHost(candidate) + if (host === null) { + this.state.skippedHosts++ + continue + } + if (hosts.has(host)) continue + if (hosts.size >= share) { + this.state.skippedHosts++ + this.state.limitsHit.add('maxHostsPerTracker') + } else { + hosts.add(host) + } + } + return Array.from(hosts) + } + + private recordTrackerFailure(error: unknown): void { + if (error instanceof LookupResourceLimitError) this.state.limitsHit.add(error.limit) + else if (!this.controller.signal.aborted) this.state.trackersFailed++ + } + + private async processTracker(tracker: string, share: number): Promise { + try { + const candidates = await this.lookup(tracker, this.controller.signal, this.consume) + if (this.abandoned) return + this.state.sources.set(tracker, this.collectTrackerHosts(candidates, share)) + } catch (error) { + this.recordTrackerFailure(error) + } finally { + this.state.trackersCompleted++ + this.emit() + } + } + + private async drainTrackers(share: number, cursor: { value: number }): Promise { + while (!this.controller.signal.aborted && cursor.value < this.trackers.length) { + await this.processTracker(this.trackers[cursor.value++], share) + } + } + + private async run(): Promise { + const cursor = { value: 0 } + // Each tracker keeps a reserved share, so an early advertisement flood + // cannot consume the complete candidate budget before later sources reply. + const share = Math.min(this.limits.maxHostsPerTracker, + Math.max(1, Math.floor(this.limits.maxHosts / Math.max(1, this.trackers.length)))) + await Promise.all(Array.from( + { length: Math.min(this.limits.trackerConcurrency, this.trackers.length) }, + () => this.drainTrackers(share, cursor) + )) + this.state.done = true + this.finish(this.state, this.abandoned) + this.emit() + this.listeners.clear() + } +} diff --git a/packages/sdk/src/overlay-tools/LookupHostQueue.ts b/packages/sdk/src/overlay-tools/LookupHostQueue.ts new file mode 100644 index 000000000..33d0f0d43 --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupHostQueue.ts @@ -0,0 +1,78 @@ +/** A bounded FIFO within each source, round-robin between sources. */ +export class LookupHostQueue { + private readonly queues = new Map() + private readonly seen = new Set() + private cursor = 0 + private active = 0 + private closed = false + private sourceClosed = false + private resolveDone: () => void = () => {} + readonly done = new Promise(resolve => { this.resolveDone = resolve }) + + constructor( + private readonly maxHosts: number, + private readonly concurrency: number, + private readonly run: (host: string) => Promise, + private readonly skipped: (count: number, limited: boolean) => void + ) {} + + add(source: string, hosts: string[]): void { + if (this.closed || this.sourceClosed) return + const queue = this.queues.get(source) ?? [] + this.queues.set(source, queue) + for (const host of hosts) { + if (this.seen.has(host)) continue + if (this.seen.size >= this.maxHosts) { this.skipped(1, true); continue } + this.seen.add(host) + queue.push(host) + } + this.pump() + } + + finishSources(): void { + this.sourceClosed = true + this.pump() + } + + cancel(): void { + if (this.closed) return + this.closed = true + for (const queue of this.queues.values()) { + this.skipped(queue.length, false) + queue.length = 0 + } + this.settle() + } + + private next(): string | undefined { + const sources = Array.from(this.queues.values()) + for (const [offset] of sources.entries()) { + const index = (this.cursor + offset) % sources.length + const host = sources[index].shift() + if (host !== undefined) { + this.cursor = index + 1 + return host + } + } + this.cursor += sources.length + return undefined + } + + private settle(): void { + if ((this.closed || this.sourceClosed) && this.active === 0 && + Array.from(this.queues.values()).every(queue => queue.length === 0)) this.resolveDone() + } + + private pump(): void { + while (!this.closed && this.active < this.concurrency) { + const host = this.next() + if (host === undefined) break + this.active++ + void this.run(host).catch(() => {}).finally(() => { + this.active-- + this.pump() + }) + } + this.settle() + } +} diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 640b5cfae..ba93f4ca8 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -1,7 +1,14 @@ +import { LookupDiscovery, LookupDiscoveryUpdate } from './LookupDiscovery.js' +import { LookupHostQueue } from './LookupHostQueue.js' +import { readLookupResponseBytes } from './LookupResponseReader.js' +import { DEFAULT_LOOKUP_LIMITS, LookupLimits, LookupResourceLimitError, lookupLimits, normalizeLookupHost, lookupAbortError, withLookupAbort } from './LookupResources.js' +export type { LookupLimits } from './LookupResources.js' +export { DEFAULT_LOOKUP_LIMITS, LookupResourceLimitError } from './LookupResources.js' import { Transaction } from '../transaction/index.js' import { Beef } from '../transaction/Beef.js' import OverlayAdminTokenTemplate from './OverlayAdminTokenTemplate.js' import * as Utils from '../primitives/utils.js' +import { sha256 } from '../primitives/Hash.js' import { getOverlayHostReputationTracker, HostReputationTracker } from './HostReputationTracker.js' import { Telemetry, TelemetryConfig } from '../telemetry/Telemetry.js' import { normalizeBRC100ByteFields, stringifyBRC100 } from '../wallet/BRC100ByteEncoding.js' @@ -54,19 +61,39 @@ export type LookupFacilitatorAnswer = LookupAnswer | LookupFreeformAnswer */ export interface LookupQueryOptions { /** - * Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue - * promptly; callback completion is not awaited and failures are isolated. - * Intake stops at the configured evidenceLimits, reporting one limit event. - * No callbacks occur after the query iterator closes. Legacy answers, host - * scheduling, timeout and reputation behavior are unchanged. + * Abort this query without cancelling discovery still owned by another query. + * `query()` and `queryDetailed()` reject with an `AbortError` once this + * signal fires: a cancelled attempt never answered the question, so it is + * never reported as an empty output list. `query$()` keeps emitting its + * terminal snapshot with `terminalReason: 'cancelled'` instead. */ - onEvidence?: (event: LookupEvidenceEvent) => void | Promise + signal?: AbortSignal /** * Callback intake budget, independent of legacy aggregation. Defaults to 512 * outputs / 16 MiB of BEEF and context bytes. Values must be positive safe * integers. Coordinate these with a downstream verifier's admission limits. + * Precedence when both this and `limits.maxEvidenceOutputs`/`maxEvidenceBytes` + * are supplied for the same call: `evidenceLimits` wins, then `limits`, then + * the resolver's configured limits, then the library defaults. */ evidenceLimits?: { maxOutputs?: number; maxBytes?: number } + /** Whole attempt budget including discovery and queued hosts. Default 10000 ms. */ + deadlineMs?: number + /** + * Per-query operational resource limits (discovery, transport and queueing + * bounds). `limits.maxEvidenceOutputs`/`maxEvidenceBytes` also set the + * evidence intake budget, but the `evidenceLimits` shorthand above takes + * precedence over these two fields when both are supplied. + */ + limits?: Partial + /** + * Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue + * promptly; callback completion is not awaited and failures are isolated. + * Intake stops at the configured evidenceLimits, reporting one limit event. + * No callbacks occur after the query iterator closes. Raw `query$` snapshots + * remain unverified transport aggregates, not cryptographic proof. + */ + onEvidence?: (event: LookupEvidenceEvent) => void | Promise /** * Override the grace window (ms) between the first valid response and the resolution of the query. * Late responders arriving within this window are merged into the result. Default 80 ms. @@ -112,7 +139,8 @@ export interface LookupQueryOptions { /** Additive evidence intake, independent of the legacy aggregated answer. */ export type LookupEvidenceEvent = - { type: 'output'; host: string; output: LookupAnswer['outputs'][number] } | { type: 'limit' } + | { type: 'output'; host: string; output: LookupAnswer['outputs'][number] } + | { type: 'limit' } /** Info supplied to onUnreachableHost callbacks. */ export interface UnreachableHostInfo { @@ -132,6 +160,20 @@ export interface UnreachableHostInfo { * and refine in place as more hosts answer. */ export interface LookupAnswerProgress { + /** Transport coverage only, never cryptographic validity or global absence. */ + discoveryComplete?: boolean + terminalReason?: 'settled' | 'deadline' | 'cancelled' | 'resource-limit' + discoveredHosts?: number + skippedHosts?: number + receivedBytes?: number + /** Retained decoded BEEF/context octets; JavaScript arrays have additional heap overhead. */ + retainedBytes?: number + /** Receipt-copy octets handed to onEvidence, independently bounded. */ + evidenceBytes?: number + trackersTotal?: number + trackersCompleted?: number + trackersFailed?: number + limitsHit?: string[] type: 'output-list' outputs: Array<{ beef: number[]; outputIndex: number; context?: number[]; txid?: string }> /** Parallel array of resolved tx ids for each output (same index as `outputs`). */ @@ -263,6 +305,20 @@ function isFreeformAnswer(value: unknown): value is LookupFreeformAnswer { return answer.type === 'freeform' && Object.hasOwn(answer, 'result') } +function lookupAnswerRetainedBytes(answer: LookupAnswer): number { + let retained = 0 + for (const output of answer.outputs) retained += output.beef.length + (output.context?.length ?? 0) + return retained +} + +function copyLookupOutput(output: LookupAnswer['outputs'][number]): LookupAnswer['outputs'][number] { + return { + ...output, + beef: output.beef.slice(), + ...(output.context === undefined ? {} : { context: output.context.slice() }) + } +} + /** A wall-clock deadline that rejects after `timeoutMs`, optionally aborting a controller. */ interface Deadline { /** Rejects with `Error('Request timed out')` once the timer fires. */ @@ -324,8 +380,70 @@ interface CacheOptions { txMemoTtlMs?: number } +/** Discovery fields that can truncate the cached SLAP host set. */ +interface LookupDiscoveryBound { + maxHosts: number + maxHostsPerTracker: number + maxTrackers: number + maxResponseBytes: number + maxTotalBytes: number + maxOutputs: number +} + +interface LookupHostsCacheEntry extends LookupDiscoveryBound { + hosts: string[] + expiresAt: number + discoveryComplete?: boolean + trackersFailed?: number + limitsHit?: string[] +} + +function lookupDiscoveryBound(limits: LookupLimits): LookupDiscoveryBound { + return { + maxHosts: limits.maxHosts, + maxHostsPerTracker: limits.maxHostsPerTracker, + maxTrackers: limits.maxTrackers, + maxResponseBytes: limits.maxResponseBytes, + maxTotalBytes: limits.maxTotalBytes, + maxOutputs: limits.maxOutputs + } +} + +/** In-flight discovery identity: service plus the limits that shape tracker work. */ +function lookupDiscoveryCacheKey(service: string, limits: LookupLimits): string { + const bound = lookupDiscoveryBound(limits) + return stringifyBRC100([ + service, + bound.maxHosts, + bound.maxHostsPerTracker, + bound.maxTrackers, + limits.trackerConcurrency, + bound.maxResponseBytes, + bound.maxTotalBytes, + bound.maxOutputs + ]) +} + +/** True when `cached` was produced with bounds at least as permissive as `needed`. */ +function lookupDiscoveryCovers( + cached: Partial | undefined, + needed: LookupDiscoveryBound +): boolean { + if (cached === undefined) return false + return ( + (cached.maxHosts ?? 0) >= needed.maxHosts && + (cached.maxHostsPerTracker ?? 0) >= needed.maxHostsPerTracker && + (cached.maxTrackers ?? 0) >= needed.maxTrackers && + (cached.maxResponseBytes ?? 0) >= needed.maxResponseBytes && + (cached.maxTotalBytes ?? 0) >= needed.maxTotalBytes && + (cached.maxOutputs ?? 0) >= needed.maxOutputs + ) +} + /** Configuration options for the Lookup resolver. */ export interface LookupResolverConfig { + /** Defaults for the bounded discovery, scheduler and receipt intake. */ + limits?: Partial /** * The network preset to use, unless other options override it. * - mainnet: use mainnet SLAP trackers and HTTPS facilitator @@ -364,10 +482,19 @@ export interface OverlayLookupFacilitator { lookup: ( url: string, question: LookupQuestion, - timeout?: number + timeout?: number, + signal?: AbortSignal, + options?: LookupRequestOptions ) => Promise } +/** Optional bounded transport settings; older custom facilitators may ignore these. */ +export interface LookupRequestOptions { + maxResponseBytes?: number + maxOutputs?: number + consumeBytes?: (bytes: number) => void +} + export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { fetchClient: typeof fetch allowHTTP: boolean @@ -386,13 +513,18 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { async lookup( url: string, question: LookupQuestion, - timeout: number = 2000 + timeout: number = 2000, + signal?: AbortSignal, + options?: LookupRequestOptions ): Promise { if (!url.startsWith('https:') && !this.allowHTTP) { throw new Error('HTTPS facilitator can only use URLs that start with "https:"') } const controller = typeof AbortController === 'undefined' ? undefined : new AbortController() + if (signal?.aborted === true) throw lookupAbortError() + const abort = (): void => controller?.abort() + signal?.addEventListener('abort', abort, { once: true }) const deadline = createDeadline(timeout, controller) // Hard wall-clock deadline: in some environments (e.g. browser/Electron CORS @@ -400,25 +532,29 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { // AbortController signal alone is insufficient to make the returned promise // resolve or reject. Race the fetch against a setTimeout-backed reject so // the consumer-facing promise always settles within `timeout` ms. - const fetchPromise = this.performLookupRequest(url, question, controller?.signal) + const fetchPromise = this.performLookupRequest(url, question, controller?.signal, options) // Swallow background rejection if the deadline wins first. fetchPromise.catch(() => { /* noop */ }) try { - return await Promise.race([fetchPromise, deadline.promise]) + return await withLookupAbort(Promise.race([fetchPromise, deadline.promise]), signal) } catch (e) { + if (signal?.aborted) throw lookupAbortError() throw normalizeLookupError(e, deadline.didTimeOut()) } finally { deadline.cancel() + signal?.removeEventListener('abort', abort) + controller?.abort() } } private async performLookupRequest( url: string, question: LookupQuestion, - signal: AbortSignal | undefined + signal: AbortSignal | undefined, + options?: LookupRequestOptions ): Promise { const fco: RequestInit = { method: 'POST', @@ -427,10 +563,20 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { 'X-Aggregation': 'yes' }, body: stringifyBRC100({ service: question.service, query: question.query }), + // normalizeLookupHost and the https: guard above validate the advertised + // URL only. A followed 307/308 would carry the serialized query body to + // an origin neither check ever saw, so an untrusted SLAP host could + // redirect a lookup (or a tracker discovery request, which uses this + // same path) to http:, loopback or link-local. Fail closed instead: the + // rejection is recorded as an ordinary availability failure for the + // advertised host. + redirect: 'error', signal } const response: Response = await this.fetchClient(`${url}/lookup`, fco) - if (!response.ok) { + if (signal?.aborted === true || !response.ok) { + try { void response.body?.cancel().catch(() => {}) } catch { /* best-effort body cleanup */ } + if (signal?.aborted === true) throw lookupAbortError() // 408/429 are availability/backpressure signals. Other 4xx responses // reject this request but do not prove that the host is unavailable, so // they remain distinguishable and neutral for availability reputation. @@ -444,10 +590,15 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { : 'semantic' throw new LookupHTTPError(response.status, kind, response.statusText) } + const payload = await readLookupResponseBytes(response, { + signal, + maxResponseBytes: options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes, + consumeBytes: options?.consumeBytes + }) if (isOctetStream(response.headers.get('content-type'))) { - return await this.parseOctetStreamLookup(response) + return await this.parseOctetStreamLookup(payload, signal, options) } - const answer = await response.json() + const answer = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(payload)) if ( answer != null && typeof answer === 'object' && @@ -455,6 +606,7 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { answer.type === 'output-list' && Array.isArray(answer.outputs) ) { + if (answer.outputs.length > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) throw new LookupResourceLimitError('maxOutputs') for (const output of answer.outputs) { normalizeBRC100ByteFields(output, ['beef', 'context']) } @@ -463,10 +615,10 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } /** Parse the aggregated octet-stream lookup response into an output-list LookupAnswer. */ - private async parseOctetStreamLookup(response: Response): Promise { - const payload = await response.arrayBuffer() - const r = new Utils.Reader([...new Uint8Array(payload)]) + private async parseOctetStreamLookup(payload: Uint8Array, signal?: AbortSignal, options?: LookupRequestOptions): Promise { + const r = new Utils.Reader(Array.from(payload)) const nOutpoints = r.readVarIntNum() + if (!Number.isSafeInteger(nOutpoints) || nOutpoints < 0 || nOutpoints > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) throw new LookupResourceLimitError('maxOutputs') const outpoints: Array<{ txid: string; outputIndex: number; context?: number[] }> = [] for (let i = 0; i < nOutpoints; i++) { const txid = Utils.toHex(r.read(32)) @@ -477,14 +629,16 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } const beef = r.read() const beefObj = Beef.fromBinary(beef) - const outputs = await this.extractAtomicOutputs(outpoints, beefObj) + const outputs = await this.extractAtomicOutputs(outpoints, beefObj, signal, options) return { type: 'output-list', outputs } } /** Memoize per-txid atomic BEEF extraction, yielding to the event loop between outputs. */ private async extractAtomicOutputs( outpoints: Array<{ txid: string; outputIndex: number; context?: number[] }>, - beefObj: Beef + beefObj: Beef, + signal?: AbortSignal, + options?: LookupRequestOptions ): Promise> { const beefByTxid = new Map() const outputs: Array<{ @@ -493,13 +647,17 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { beef: number[] txid: string }> = Array.from({ length: outpoints.length }) + let extractedBytes = 0 for (let idx = 0; idx < outpoints.length; idx++) { + if (signal?.aborted === true) throw lookupAbortError() const x = outpoints[idx] let beefBytes = beefByTxid.get(x.txid) if (beefBytes === undefined) { beefBytes = beefObj.toBinaryAtomic(x.txid) beefByTxid.set(x.txid, beefBytes) } + extractedBytes += beefBytes.length + (x.context?.length ?? 0) + if (extractedBytes > (options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes)) throw new LookupResourceLimitError('maxResponseBytes') outputs[idx] = { outputIndex: x.outputIndex, context: x.context, @@ -515,23 +673,21 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } } -type LookupQueryEvent = - { kind: 'answer'; answer: LookupAnswer } | { kind: 'done' } | { kind: 'grace' } | { kind: 'soft' } - interface LookupQuerySessionOptions { - hostCount: number graceMs: number softTimeoutMs?: number waitForAllHosts: boolean correlationId?: string - evidenceLimits?: { maxOutputs?: number; maxBytes?: number } + limits: LookupLimits + onEvidence: LookupQueryOptions['onEvidence'] resolveTxId: (output: LookupAnswer['outputs'][number], now: number) => string | null } +/** A single cumulative snapshot plus a wake flag, regardless of listener speed. */ class LookupQuerySession { readonly startedAt = Date.now() - readonly hostCount: number readonly correlationId?: string + hostCount = 0 completedHosts = 0 successfulHosts = 0 emptyHosts = 0 @@ -540,236 +696,157 @@ class LookupQuerySession { freeformHosts = 0 emittedFinal = false closed = false + accepting = true + discoveryComplete = false + discoveredHosts = 0 + skippedHosts = 0 + receivedBytes = 0 + retainedBytes = 0 + trackersTotal = 0 + trackersCompleted = 0 + trackersFailed = 0 + readonly limitsHit = new Set() + terminalReason: NonNullable = 'settled' private evidenceOutputs = 0 private evidenceBytes = 0 private evidenceLimited = false - private readonly maxEvidenceOutputs: number - private readonly maxEvidenceBytes: number - - private readonly graceMs: number - private readonly softTimeoutMs?: number - private readonly waitForAllHosts: boolean - private readonly resolveTxId: LookupQuerySessionOptions['resolveTxId'] - private readonly outputsMap = new Map< - string, - { beef: number[]; context?: number[]; outputIndex: number } - >() + private limitNotificationSent = false + private readonly outputsMap = new Map() private readonly txIds: string[] = [] - private readonly queue: LookupQueryEvent[] = [] private waiter: (() => void) | null = null + private dirty = false + private finished = false + private failure: unknown private firstResponseAt: number | null = null private graceTimer: ReturnType | null = null private softTimer: ReturnType | null = null private graceFired = false - private emittedOnce = false + private softFired = false - constructor(options: LookupQuerySessionOptions) { - this.maxEvidenceOutputs = options.evidenceLimits?.maxOutputs ?? 512 - this.maxEvidenceBytes = options.evidenceLimits?.maxBytes ?? 16 * 1024 * 1024 - if ( - ![this.maxEvidenceOutputs, this.maxEvidenceBytes].every( - value => Number.isSafeInteger(value) && value > 0 - ) - ) - throw new Error('Evidence intake limits must be positive safe integers') - this.hostCount = options.hostCount - this.graceMs = options.graceMs - this.softTimeoutMs = options.softTimeoutMs - this.waitForAllHosts = options.waitForAllHosts + constructor(private readonly options: LookupQuerySessionOptions) { this.correlationId = options.correlationId - this.resolveTxId = options.resolveTxId } - private push(event: LookupQueryEvent): void { - this.queue.push(event) - if (this.waiter === null) return + wake(): void { + this.dirty = true const waiter = this.waiter this.waiter = null - waiter() + waiter?.() } - recordOutputAnswer(answer: LookupAnswer): void { - this.successfulHosts++ - if (answer.outputs.length === 0) { - this.emptyHosts++ - return + limit(name: string): void { + this.limitsHit.add(name) + if (!this.limitNotificationSent && this.accepting) { + this.limitNotificationSent = true + try { void Promise.resolve(this.options.onEvidence?.({ type: 'limit' })).catch(() => {}) } catch { /* consumer isolation */ } } - this.push({ kind: 'answer', answer }) + if (this.terminalReason === 'settled') this.terminalReason = 'resource-limit' + this.wake() } - receiveEvidence( - host: string, - answer: LookupAnswer, - callback: LookupQueryOptions['onEvidence'] - ): void { - if (callback === undefined || this.closed || this.evidenceLimited) return + receiveEvidence(host: string, answer: LookupAnswer, callback: LookupQueryOptions['onEvidence']): void { + if (callback === undefined || this.closed || !this.accepting || this.evidenceLimited) return const deliver = (event: LookupEvidenceEvent): void => { - try { - void Promise.resolve(callback(event)).catch(() => {}) - } catch { - /* consumer isolation */ - } + try { void Promise.resolve(callback(event)).catch(() => {}) } catch { /* consumer isolation */ } } for (const output of answer.outputs) { + if (!this.accepting || this.closed) break const bytes = output.beef.length + (output.context?.length ?? 0) - if ( - this.evidenceOutputs >= this.maxEvidenceOutputs || - this.evidenceBytes + bytes > this.maxEvidenceBytes - ) { + if (this.evidenceOutputs >= this.options.limits.maxEvidenceOutputs || + bytes > this.options.limits.maxEvidenceBytes - this.evidenceBytes) { this.evidenceLimited = true - deliver({ type: 'limit' }) + this.limit(this.evidenceOutputs >= this.options.limits.maxEvidenceOutputs ? 'maxEvidenceOutputs' : 'maxEvidenceBytes') break } this.evidenceOutputs++ this.evidenceBytes += bytes - deliver({ - type: 'output', - host, - output: { - ...output, - beef: output.beef.slice(), - ...(output.context === undefined ? {} : { context: output.context.slice() }) - } - }) + deliver({ type: 'output', host, output: { + ...output, beef: output.beef.slice(), + ...(output.context === undefined ? {} : { context: output.context.slice() }) + } }) } } - recordFreeformAnswer(): void { - this.freeformHosts++ - } - - recordRejection(): void { - this.rejectedHosts++ - } - - recordAvailabilityFailure(): void { - this.failedHosts++ + recordOutputAnswer(answer: LookupAnswer): void { + if (this.closed || !this.accepting) return + this.successfulHosts++ + if (answer.outputs.length === 0) { this.emptyHosts++; return } + this.mergeAnswer(answer) + if (this.firstResponseAt === null) { + this.firstResponseAt = Date.now() + if (this.options.graceMs > 0) this.graceTimer = setTimeout(() => { + this.graceFired = true; this.wake() + }, this.options.graceMs) + else this.graceFired = true + } + this.wake() } - recordDone(): void { - this.completedHosts++ - this.push({ kind: 'done' }) - } + recordFreeformAnswer(): void { if (!this.closed) this.freeformHosts++ } + recordRejection(): void { if (!this.closed) this.rejectedHosts++ } + recordAvailabilityFailure(): void { if (!this.closed) this.failedHosts++ } + recordDone(): void { if (!this.closed) { this.completedHosts++; this.wake() } } - private mergeAnswer(answer: LookupAnswer): boolean { - let added = false + private mergeAnswer(answer: LookupAnswer): void { const now = Date.now() for (const output of answer.outputs) { - const txId = this.resolveTxId(output, now) + const txId = this.options.resolveTxId(output, now) if (txId === null) continue const key = `${txId}.${output.outputIndex}` if (this.outputsMap.has(key)) continue + if (this.outputsMap.size >= this.options.limits.maxOutputs) { this.limit('maxOutputs'); break } this.outputsMap.set(key, output) this.txIds.push(txId) - added = true } - return added } + finish(error?: unknown): void { this.failure = error; this.finished = true; this.wake() } + snapshot(isFinal: boolean): LookupAnswerProgress { return { - type: 'output-list', - outputs: Array.from(this.outputsMap.values()), - txIds: this.txIds.slice(), - isFinal, - hostCount: this.hostCount, - completedHosts: this.completedHosts, - successfulHosts: this.successfulHosts, - emptyHosts: this.emptyHosts, - failedHosts: this.failedHosts, - rejectedHosts: this.rejectedHosts, - freeformHosts: this.freeformHosts, + type: 'output-list', outputs: Array.from(this.outputsMap.values()), txIds: this.txIds.slice(), + isFinal, hostCount: this.hostCount, completedHosts: this.completedHosts, + successfulHosts: this.successfulHosts, emptyHosts: this.emptyHosts, failedHosts: this.failedHosts, + rejectedHosts: this.rejectedHosts, freeformHosts: this.freeformHosts, + discoveryComplete: this.discoveryComplete, + ...(isFinal ? { terminalReason: this.terminalReason } : {}), + discoveredHosts: this.discoveredHosts, skippedHosts: this.skippedHosts, + receivedBytes: this.receivedBytes, retainedBytes: this.retainedBytes, evidenceBytes: this.evidenceBytes, trackersTotal: this.trackersTotal, + trackersCompleted: this.trackersCompleted, trackersFailed: this.trackersFailed, + limitsHit: Array.from(this.limitsHit), ...(this.correlationId !== undefined ? { correlationId: this.correlationId } : {}) } } - private handleAnswer(answer: LookupAnswer): LookupAnswerProgress | null { - const added = this.mergeAnswer(answer) - if (this.firstResponseAt === null) { - this.firstResponseAt = Date.now() - if (!this.graceFired && this.graceMs > 0) { - this.graceTimer = setTimeout(() => { - this.graceFired = true - this.push({ kind: 'grace' }) - }, this.graceMs) - } else { - this.graceFired = true - } - } - if (this.graceFired && added && (this.emittedOnce || !this.waitForAllHosts)) { - this.emittedOnce = true - return this.snapshot(false) - } - return null - } - - private handleGrace(): LookupAnswerProgress | null { - if (this.emittedOnce || this.waitForAllHosts) return null - this.emittedOnce = true - return this.snapshot(false) - } - - private handleSoft(): { - snapshot: LookupAnswerProgress | null - stop: boolean - } { - let snapshot: LookupAnswerProgress | null = null - if (!this.emittedOnce) { - this.graceFired = true - this.emittedOnce = true - snapshot = this.snapshot(false) - } - return { - snapshot, - stop: typeof this.softTimeoutMs === 'number' && this.firstResponseAt !== null - } - } - - private async nextEvent(): Promise { - if (this.queue.length === 0) { - await new Promise(resolve => { - this.waiter = resolve - }) - } - return this.queue.shift() as LookupQueryEvent - } - - private processEvent(event: LookupQueryEvent): { - snapshot: LookupAnswerProgress | null - stop: boolean - } { - switch (event.kind) { - case 'answer': - return { snapshot: this.handleAnswer(event.answer), stop: false } - case 'grace': - return { snapshot: this.handleGrace(), stop: false } - case 'soft': - return this.handleSoft() - case 'done': - return { snapshot: null, stop: false } - } + close(): void { + this.closed = true + this.accepting = false + if (this.graceTimer !== null) clearTimeout(this.graceTimer) + if (this.softTimer !== null) clearTimeout(this.softTimer) + this.wake() } async *progress(): AsyncIterable { - if (typeof this.softTimeoutMs === 'number' && this.softTimeoutMs >= 0) { - this.softTimer = setTimeout(() => this.push({ kind: 'soft' }), this.softTimeoutMs) + if (typeof this.options.softTimeoutMs === 'number' && this.options.softTimeoutMs >= 0) { + this.softTimer = setTimeout(() => { this.softFired = true; this.wake() }, this.options.softTimeoutMs) } try { - let stop = false - while (this.completedHosts < this.hostCount && !stop) { - const event = await this.nextEvent() - const outcome = this.processEvent(event) - if (outcome.snapshot != null) yield outcome.snapshot - stop = outcome.stop + while (!this.closed) { + if (this.finished) { + if (this.failure !== undefined) throw this.failure + this.emittedFinal = true + yield this.snapshot(true) + return + } + if (this.dirty && (this.softFired || (this.graceFired && !this.options.waitForAllHosts))) { + this.dirty = false + yield this.snapshot(false) + } else { + this.dirty = false + await new Promise(resolve => { this.waiter = resolve }) + } } - const finalSnapshot = this.snapshot(true) - this.emittedFinal = true - yield finalSnapshot - } finally { - this.closed = true - if (this.graceTimer !== null) clearTimeout(this.graceTimer) - if (this.softTimer !== null) clearTimeout(this.softTimer) - } + } finally { this.close() } } } @@ -783,6 +860,38 @@ interface LookupHostFailureContext { notificationCooldownMs: number } +/** Mutable orchestration state for one raw lookup query. */ +interface LookupQueryRun { + question: LookupQuestion + timeout: number | undefined + options: LookupQueryOptions | undefined + limits: LookupLimits + session: LookupQuerySession + controller: AbortController + queue: LookupHostQueue + seen: Set + sourceQuota: number + discoveryBytes: number + discoverySkipped: number + processedSources: Set + releaseDiscovery: (() => void) | undefined + discoveryFinished: boolean + noHostsError: Error | undefined + cleaned: boolean + timer: ReturnType + abort: () => void + iteratorSignal: AbortSignal +} + +interface LookupDiscoveryPlan { + key: string + cached: LookupHostsCacheEntry | undefined + configuredAdditional: string[] + cacheHasAvailableHost: boolean + refresh: boolean + initialQuota: number +} + /** * Represents a Lookup Resolver. */ @@ -796,8 +905,11 @@ export default class LookupResolver { private readonly telemetry: Telemetry // ---- Caches / memoization ---- - private readonly hostsCache: Map - private readonly hostsInFlight: Map> + private readonly hostsCache: Map + private readonly hostsInFlight: Map + private readonly limits: LookupLimits + private activeQueries = 0 + private trackerCursor = 0 private readonly hostsTtlMs: number private readonly hostsMaxEntries: number @@ -813,6 +925,7 @@ export default class LookupResolver { private readonly lastUnreachableNotificationAt: Map constructor(config: LookupResolverConfig = {}) { + this.limits = lookupLimits(config.limits) this.networkPreset = config.networkPreset ?? 'mainnet' this.facilitator = config.facilitator ?? @@ -869,6 +982,9 @@ export default class LookupResolver { * Optional `options.graceMs` overrides the per-call grace window (default 80 ms). * Optional `options.softTimeoutMs` resolves the query early with whatever has arrived once any host has * answered (or with an empty result if no host has answered by `softTimeoutMs`). + * + * Throws an `AbortError` when `options.signal` aborted the attempt, so a + * cancelled lookup is never mistaken for an authoritative empty answer. */ async query( question: LookupQuestion, @@ -882,6 +998,14 @@ export default class LookupResolver { * Performs a lookup and returns both its answer and the host settlement * evidence required by security-sensitive consumers to distinguish an * authoritative empty result from an availability failure. + * + * Throws an `AbortError` when `options.signal` aborted the attempt, rather + * than returning a resolution whose empty answer would have to be + * re-qualified against `progress.terminalReason`. When a client resource + * budget was exhausted during SLAP discovery, before any host could be + * admitted, it throws `LookupResourceLimitError` naming that limit; the + * historical no-competent-hosts error is reserved for a deadline or a + * settled attempt that genuinely found no host. */ async queryDetailed( question: LookupQuestion, @@ -906,91 +1030,54 @@ export default class LookupResolver { } finally { await iter.return?.(undefined) } - return { - answer: { - type: 'output-list', - outputs: last?.outputs ?? [] - }, - progress: last ?? { - type: 'output-list', - outputs: [], - txIds: [], - isFinal: true, - hostCount: 0, - completedHosts: 0, - successfulHosts: 0, - emptyHosts: 0, - failedHosts: 0, - rejectedHosts: 0, - freeformHosts: 0, - ...(options?.correlationId !== undefined ? { correlationId: options.correlationId } : {}) - } - } - } - - private appendAdditionalHosts(service: string, hosts: string[]): void { - const additional = this.additionalHosts[service] - if (additional == null || additional.length === 0) return - const seen = new Set(hosts) - for (const host of additional) { - if (!seen.has(host)) hosts.push(host) - } - } - - private async competentHostsFor(question: LookupQuestion): Promise { - let hosts: string[] - if (question.service === 'ls_slap') { - hosts = this.networkPreset === 'local' ? ['http://localhost:8080'] : this.slapTrackers - } else if (this.hostOverrides[question.service] != null) { - hosts = this.hostOverrides[question.service] - } else if (this.networkPreset === 'local') { - hosts = ['http://localhost:8080'] - } else { - hosts = await this.getCompetentHostsCached(question.service) + const progress: LookupAnswerProgress = last ?? { + type: 'output-list', + outputs: [], + txIds: [], + isFinal: true, + hostCount: 0, + completedHosts: 0, + successfulHosts: 0, + emptyHosts: 0, + failedHosts: 0, + rejectedHosts: 0, + freeformHosts: 0, + terminalReason: 'settled', + ...(options?.correlationId !== undefined ? { correlationId: options.correlationId } : {}) } - this.appendAdditionalHosts(question.service, hosts) - if (hosts.length < 1) { + // Promise callers cannot see terminalReason. A cancelled attempt never + // answered the question, so it must not flatten into an empty output list + // at any host count. + if (progress.terminalReason === 'cancelled') throw lookupAbortError() + // A deadline that admitted no host is a miss, not a successful empty + // answer from a queried host. An attempt that exhausted a client resource + // budget during discovery is a third outcome: the trackers were never + // given the chance to name a host, so it keeps its own error and limit + // rather than borrowing the no-competent-hosts message. + if (progress.hostCount === 0) { + if (progress.terminalReason === 'resource-limit') { + throw new LookupResourceLimitError(progress.limitsHit?.[0] ?? 'resource-limit') + } throw new Error( `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` ) } - return hosts - } - - private isSlapRecoveryEligible(service: string): boolean { - return ( - service !== 'ls_slap' && this.hostOverrides[service] == null && this.networkPreset !== 'local' - ) - } - - private async rankedHostsFor(question: LookupQuestion): Promise { - const competentHosts = await this.competentHostsFor(question) - let rankedHosts: string[] - try { - rankedHosts = this.prepareHostsForQuery(competentHosts, `lookup service ${question.service}`) - } catch (error) { - if (!this.isSlapRecoveryEligible(question.service)) throw error - this.hostsCache.delete(question.service) - const fresh = await this.refreshHosts(question.service, true) - this.appendAdditionalHosts(question.service, fresh) - if (fresh.length < 1) { - throw new Error( - `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` - ) - } - rankedHosts = this.prepareHostsForQuery(fresh, `lookup service ${question.service}`) - } - if (rankedHosts.length < 1) { - throw new Error( - `All competent hosts for ${question.service} are temporarily unavailable due to backoff.` - ) + return { + answer: { + type: 'output-list', + outputs: progress.outputs + }, + progress } - return rankedHosts } - private unreachableNotificationCooldown(options: LookupQueryOptions | undefined): number { + private unreachableNotificationCooldown( + options: LookupQueryOptions | undefined + ): number { const requested = options?.unreachableHostNotificationCooldownMs - return typeof requested === 'number' && Number.isFinite(requested) && requested >= 0 + return typeof requested === 'number' && + Number.isFinite(requested) && + requested >= 0 ? requested : DEFAULT_UNREACHABLE_NOTIFICATION_COOLDOWN_MS } @@ -1006,9 +1093,13 @@ export default class LookupResolver { const notificationKey = `${service}\u0000${host}` const now = Date.now() const lastNotificationAt = - this.lastUnreachableNotificationAt.get(notificationKey) ?? Number.NEGATIVE_INFINITY + this.lastUnreachableNotificationAt.get(notificationKey) ?? + Number.NEGATIVE_INFINITY if (now - lastNotificationAt < cooldownMs) return - if (this.lastUnreachableNotificationAt.size >= MAX_NOTIFICATION_DEDUP_ENTRIES) { + if ( + this.lastUnreachableNotificationAt.size >= + MAX_NOTIFICATION_DEDUP_ENTRIES + ) { this.evictOldest(this.lastUnreachableNotificationAt) } this.lastUnreachableNotificationAt.set(notificationKey, now) @@ -1047,10 +1138,19 @@ export default class LookupResolver { return } session.recordFreeformAnswer() - this.captureHostTelemetry(service, host, 'freeform', Date.now() - hostStartedAt, correlationId) + this.captureHostTelemetry( + service, + host, + 'freeform', + Date.now() - hostStartedAt, + correlationId + ) } - private recordLookupHostFailure(context: LookupHostFailureContext, error: unknown): void { + private recordLookupHostFailure( + context: LookupHostFailureContext, + error: unknown + ): void { const { session, service, @@ -1072,192 +1172,582 @@ export default class LookupResolver { error ) if (!semanticRejection) { - this.notifyUnreachableHost(host, service, error, onUnreachableHost, notificationCooldownMs) + this.notifyUnreachableHost( + host, + service, + error, + onUnreachableHost, + notificationCooldownMs + ) } } - private startLookupHostQueries( - hosts: string[], - question: LookupQuestion, - timeout: number | undefined, - session: LookupQuerySession, - options: LookupQueryOptions | undefined - ): void { - const correlationId = session.correlationId - const notificationCooldownMs = this.unreachableNotificationCooldown(options) - for (const host of hosts) { - const hostStartedAt = Date.now() - void this.lookupHostWithTracking(host, question, timeout) - .then(answer => { - if (isOutputListAnswer(answer)) session.receiveEvidence(host, answer, options?.onEvidence) - this.recordLookupHostAnswer( - session, - question.service, - host, - answer, - hostStartedAt, - correlationId - ) - }) - .catch(error => { - this.recordLookupHostFailure( - { - session, - service: question.service, - host, - hostStartedAt, - correlationId, - onUnreachableHost: options?.onUnreachableHost, - notificationCooldownMs - }, - error - ) - }) - .finally(() => { - session.recordDone() - }) + /** + * Cumulative unverified results. Discovery remains subscribed while trackers + * settle; each new host enters the bounded queue immediately. Caller abort, + * deadline and iterator close release this query's ownership. + */ + query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable { + const cancellation = new AbortController() + const iterator = this.queryProgress(question, timeout, options, cancellation.signal)[Symbol.asyncIterator]() + return { + [Symbol.asyncIterator]: () => ({ + next: async () => await iterator.next(), + return: async () => { + cancellation.abort() + return await iterator.return?.() ?? { done: true, value: undefined } + }, + throw: async (error?: unknown) => { + cancellation.abort() + if (iterator.throw !== undefined) return await iterator.throw(error) + throw error + } + }) } } + private cloneLookupQuestion(inputQuestion: LookupQuestion): LookupQuestion { + try { + return structuredClone(inputQuestion) + } catch { + if (this.facilitator instanceof HTTPSOverlayLookupFacilitator) { + return JSON.parse(stringifyBRC100(inputQuestion)) as LookupQuestion + } + return { ...inputQuestion } + } + } + + private lookupQueryLimits(options: LookupQueryOptions | undefined): LookupLimits { + return lookupLimits(this.limits, options?.limits, this.evidenceLimitOverrides(options?.evidenceLimits)) + } + /** - * Iterable form of {@link query}. Emits partial results as hosts answer. - * - * Emission order: - * - First emission: after the grace window expires (or as soon as the soft timeout elapses), containing - * every output gathered from hosts that answered by then. - * - Subsequent emissions: re-emitted whenever a late host returns extra outputs that weren't in earlier - * emissions. Each emission contains the cumulative `outputs` set. - * - Final emission: `isFinal: true` once all in-flight hosts have settled (success / fail / timeout). The - * caller can `break` early; outstanding work is bounded by the per-host timeout. - * - * No host work runs past its per-host `timeout` — there is no leak risk on early break. + * Maps the `evidenceLimits` shorthand onto the unified `LookupLimits` + * evidence fields, validating eagerly (before any host is queried) so a + * misconfigured call fails the same way regardless of which spelling was + * used. `evidenceLimits` takes precedence over `options.limits` when both + * set the same field; see {@link LookupQueryOptions.evidenceLimits}. */ - async *query$( + private evidenceLimitOverrides( + evidenceLimits: LookupQueryOptions['evidenceLimits'] + ): Partial { + if (evidenceLimits === undefined) return {} + const maxEvidenceOutputs = evidenceLimits.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxEvidenceOutputs + const maxEvidenceBytes = evidenceLimits.maxBytes ?? DEFAULT_LOOKUP_LIMITS.maxEvidenceBytes + if ( + ![maxEvidenceOutputs, maxEvidenceBytes].every( + value => Number.isSafeInteger(value) && value > 0 + ) + ) { + throw new Error('Evidence intake limits must be positive safe integers') + } + return { maxEvidenceOutputs, maxEvidenceBytes } + } + + private assertLookupDeadline(deadlineMs: number): void { + if (!Number.isFinite(deadlineMs) || deadlineMs < 0 || deadlineMs > 2_147_483_647) { + throw new RangeError('Lookup deadlineMs must be between 0 and 2147483647') + } + } + + private lookupQueryStopped(run: LookupQueryRun): boolean { + return run.controller.signal.aborted || run.session.closed + } + + private consumeLookupQueryBytes(run: LookupQueryRun, bytes: number): void { + if (run.controller.signal.aborted) throw lookupAbortError() + if (bytes > run.limits.maxTotalBytes - run.session.receivedBytes) { + run.session.limit('maxTotalBytes') + throw new LookupResourceLimitError('maxTotalBytes') + } + run.session.receivedBytes += bytes + } + + private skipQueuedLookupHosts(run: LookupQueryRun, count: number, limited: boolean): void { + if (count <= 0) return + run.session.skippedHosts += count + if (limited) run.session.limit('maxHosts') + } + + private stopLookupQueryRun(run: LookupQueryRun, reason: 'deadline' | 'cancelled'): void { + if (run.controller.signal.aborted) return + run.session.limit(reason) + run.session.terminalReason = reason + run.session.accepting = false + run.session.discoveryComplete = false + run.controller.abort() + run.releaseDiscovery?.() + run.discoveryFinished = true + run.queue.cancel() + } + + private cleanupLookupQueryRun(run: LookupQueryRun): void { + if (run.cleaned) return + run.cleaned = true + run.session.close() + clearTimeout(run.timer) + run.options?.signal?.removeEventListener('abort', run.abort) + run.iteratorSignal.removeEventListener('abort', run.abort) + run.controller.abort() + run.releaseDiscovery?.() + run.queue.cancel() + this.activeQueries-- + } + + private finishLookupSources(run: LookupQueryRun): void { + run.discoveryFinished = true + run.queue.finishSources() + } + + private lookupQueryNoHostsError(run: LookupQueryRun): Error | undefined { + if (run.session.hostCount !== 0 || run.session.terminalReason !== 'settled') return undefined + if (run.noHostsError !== undefined) return run.noHostsError + return new Error( + `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${run.question.service}` + ) + } + + private finishLookupQueryRun(run: LookupQueryRun): void { + if (!run.discoveryFinished) return + run.session.finish(this.lookupQueryNoHostsError(run)) + } + + private createLookupQueryRun( question: LookupQuestion, - timeout?: number, - options?: LookupQueryOptions - ): AsyncIterable { - const rankedHosts = await this.rankedHostsFor(question) - const hostCount = rankedHosts.length - const correlationId = - options?.correlationId ?? - (this.telemetry.enabled ? this.telemetry.createCorrelationId() : undefined) + timeout: number | undefined, + options: LookupQueryOptions | undefined, + limits: LookupLimits, + iteratorSignal: AbortSignal, + deadlineMs: number + ): LookupQueryRun { + const controller = new AbortController() const session = new LookupQuerySession({ - evidenceLimits: options?.evidenceLimits, - hostCount, graceMs: options?.graceMs ?? 80, softTimeoutMs: options?.softTimeoutMs, waitForAllHosts: options?.waitForAllHosts ?? options?.holdForUnknownHosts ?? false, - correlationId, + correlationId: options?.correlationId ?? (this.telemetry.enabled ? this.telemetry.createCorrelationId() : undefined), + limits, + onEvidence: options?.onEvidence, resolveTxId: (output, now) => this.resolveTxIdForOutput(output, now) }) + let run: LookupQueryRun + run = { + question, + timeout, + options, + limits, + session, + controller, + seen: new Set(), + sourceQuota: limits.maxHosts, + discoveryBytes: 0, + discoverySkipped: 0, + processedSources: new Set(), + releaseDiscovery: undefined, + discoveryFinished: false, + noHostsError: undefined, + cleaned: false, + iteratorSignal, + abort: () => this.stopLookupQueryRun(run, 'cancelled'), + queue: new LookupHostQueue( + limits.maxHosts, + limits.hostConcurrency, + async host => await this.runQueuedLookupHost(run, host), + (count, limited) => this.skipQueuedLookupHosts(run, count, limited) + ), + timer: setTimeout(() => this.stopLookupQueryRun(run, 'deadline'), deadlineMs) + } + options?.signal?.addEventListener('abort', run.abort, { once: true }) + iteratorSignal.addEventListener('abort', run.abort, { once: true }) + return run + } - this.telemetry.capture({ - name: 'sdk.overlay.lookup.started', - component: 'sdk.lookup-resolver', - severity: 'debug', - correlationId, - attributes: { - service: question.service, - network: this.networkPreset, - hostCount - } - }) + private retainLookupHostAnswer( + run: LookupQueryRun, + host: string, + answer: LookupFacilitatorAnswer + ): LookupFacilitatorAnswer { + if (!isOutputListAnswer(answer)) return answer + const retained = lookupAnswerRetainedBytes(answer) + if (retained > run.limits.maxTotalBytes - run.session.retainedBytes) { + throw new LookupResourceLimitError('maxTotalBytes') + } + run.session.retainedBytes += retained + const ownedAnswer: LookupAnswer = { + type: 'output-list', + outputs: answer.outputs.map(copyLookupOutput) + } + run.session.receiveEvidence(host, ownedAnswer, run.options?.onEvidence) + return ownedAnswer + } + + private recordQueuedLookupHostFailure( + run: LookupQueryRun, + host: string, + startedAt: number, + error: unknown + ): void { + if (this.lookupQueryStopped(run)) return + if (error instanceof LookupResourceLimitError) { + run.session.limit(error.limit) + return + } + this.recordLookupHostFailure({ + session: run.session, + service: run.question.service, + host, + hostStartedAt: startedAt, + correlationId: run.session.correlationId, + onUnreachableHost: run.options?.onUnreachableHost, + notificationCooldownMs: this.unreachableNotificationCooldown(run.options) + }, error) + } - this.startLookupHostQueries(rankedHosts, question, timeout, session, options) + private async settleQueuedLookupHost(run: LookupQueryRun, host: string, startedAt: number): Promise { + const answer = await this.lookupHostWithTracking(host, run.question, run.timeout, run.controller.signal, { + maxResponseBytes: run.limits.maxResponseBytes, + maxOutputs: run.limits.maxOutputs, + consumeBytes: bytes => this.consumeLookupQueryBytes(run, bytes) + }) + if (this.lookupQueryStopped(run)) return + const ownedAnswer = this.retainLookupHostAnswer(run, host, answer) + if (this.lookupQueryStopped(run)) return + this.recordLookupHostAnswer( + run.session, + run.question.service, + host, + ownedAnswer, + startedAt, + run.session.correlationId + ) + } + private async runQueuedLookupHost(run: LookupQueryRun, host: string): Promise { + if (run.controller.signal.aborted) return + run.session.hostCount++ + const startedAt = Date.now() try { - for await (const progress of session.progress()) { - if (progress.isFinal) { - this.captureLookupCompletedTelemetry( - question.service, - progress, - Date.now() - session.startedAt - ) - } - yield progress - } + await this.settleQueuedLookupHost(run, host, startedAt) + } catch (error) { + this.recordQueuedLookupHostFailure(run, host, startedAt, error) } finally { - if (!session.emittedFinal) { - this.telemetry.capture({ - name: 'sdk.overlay.lookup.cancelled', - component: 'sdk.lookup-resolver', - severity: 'debug', - correlationId, - attributes: { - service: question.service, - hostCount, - completedHosts: session.completedHosts, - durationMs: Date.now() - session.startedAt - } - }) + run.session.recordDone() + } + } + + private collectAdmittedLookupHosts(run: LookupQueryRun, source: string, candidates: string[]): string[] { + const hosts: string[] = [] + const scanLimit = Math.min(candidates.length, run.limits.maxHosts * 4) + if (candidates.length > scanLimit) { + run.session.skippedHosts += candidates.length - scanLimit + run.session.limit('maxHosts') + } + const allowParameters = source === 'configured' || source === 'additional' + for (const candidate of candidates.slice(0, scanLimit)) { + const host = normalizeLookupHost(candidate, allowParameters) + if (host === null) { + run.session.skippedHosts++ + continue + } + if (run.seen.has(host)) continue + if (run.seen.size >= run.limits.maxHosts) { + run.session.skippedHosts++ + run.session.limit('maxHosts') + continue } + run.seen.add(host) + run.session.discoveredHosts++ + hosts.push(host) } + return hosts } - /** - * Cached wrapper for competent host discovery with stale-while-revalidate. - */ - private async getCompetentHostsCached(service: string): Promise { - const now = Date.now() - const cached = this.hostsCache.get(service) - - // if fresh, return immediately - if (typeof cached === 'object' && cached.expiresAt > now) { - return cached.hosts.slice() - } - - // if stale but present, kick off a refresh if not already in-flight and return stale - if (typeof cached === 'object' && cached.expiresAt <= now) { - if (!this.hostsInFlight.has(service)) { - this.hostsInFlight.set( - service, - this.refreshHosts(service).finally(() => { - this.hostsInFlight.delete(service) - }) - ) + private admitLookupHosts(run: LookupQueryRun, source: string, candidates: string[]): void { + if (run.controller.signal.aborted) return + const hosts = this.collectAdmittedLookupHosts(run, source, candidates) + if (hosts.length === 0) return + try { + const available = this.prepareHostsForQuery(hosts, `lookup service ${run.question.service}`) + run.session.skippedHosts += hosts.length - available.length + run.queue.add(source, available) + } catch (error) { + run.session.skippedHosts += hosts.length + run.noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) + } + } + + private admitQuotaLimitedLookupHosts( + run: LookupQueryRun, + source: string, + hosts: string[], + quota: number, + limitName: string + ): void { + this.admitLookupHosts(run, source, hosts.slice(0, quota)) + if (hosts.length > quota) { + run.session.skippedHosts += hosts.length - quota + run.session.limit(limitName) + } + } + + private configuredLookupHosts(question: LookupQuestion): string[] { + if (question.service === 'ls_slap') { + if (this.networkPreset === 'local') return ['http://localhost:8080'] + return this.slapTrackers + } + return this.hostOverrides[question.service] ?? ['http://localhost:8080'] + } + + private admitConfiguredLookupSources(run: LookupQueryRun): void { + this.admitLookupHosts(run, 'configured', this.configuredLookupHosts(run.question)) + this.admitLookupHosts(run, 'additional', this.additionalHosts[run.question.service] ?? []) + run.session.discoveryComplete = true + this.finishLookupSources(run) + } + + private lookupCacheHasAvailableHost(cached: LookupHostsCacheEntry | undefined): boolean { + if (cached === undefined) return false + return cached.hosts.some(host => (this.hostReputation.snapshot(host)?.backoffUntil ?? 0) <= Date.now()) + } + + private planLookupDiscovery(run: LookupQueryRun): LookupDiscoveryPlan { + const cached = this.hostsCache.get(run.question.service) + const configuredAdditional = this.additionalHosts[run.question.service] ?? [] + const cacheHasAvailableHost = this.lookupCacheHasAvailableHost(cached) + const cacheCoversCaller = lookupDiscoveryCovers(cached, run.limits) + const cacheFresh = cached !== undefined && cached.expiresAt > Date.now() + const key = lookupDiscoveryCacheKey(run.question.service, run.limits) + const discovery = this.hostsInFlight.get(key) + const refresh = + discovery !== undefined || + cached === undefined || + !cacheCoversCaller || + !cacheFresh || + !cacheHasAvailableHost + const initialSources = + Number(cached !== undefined && cacheHasAvailableHost) + Number(configuredAdditional.length > 0) + const trackerShare = Math.max(1, Math.min(this.slapTrackers.length, run.limits.maxTrackers)) + const initialQuota = refresh + ? Math.max(1, Math.floor(run.limits.maxHosts / (initialSources + trackerShare))) + : run.limits.maxHosts + return { key, cached, configuredAdditional, cacheHasAvailableHost, refresh, initialQuota } + } + + private reuseCachedLookupDiscovery(run: LookupQueryRun, cached: LookupHostsCacheEntry | undefined): void { + run.session.discoveryComplete = cached?.discoveryComplete ?? true + run.session.trackersFailed = cached?.trackersFailed ?? 0 + for (const name of cached?.limitsHit ?? []) run.session.limit(name) + this.finishLookupSources(run) + } + + private selectSlapTrackers(run: LookupQueryRun): { trackers: string[], normalized: string[] } { + const scan = Math.min(this.slapTrackers.length, run.limits.maxTrackers) + const selected = Array.from( + { length: scan }, + (_unused, offset) => this.slapTrackers[(this.trackerCursor + offset) % this.slapTrackers.length] + ) + this.trackerCursor = (this.trackerCursor + scan) % Math.max(1, this.slapTrackers.length) + const normalized = Array.from(new Set(selected.map(host => normalizeLookupHost(host)).filter((host): host is string => host !== null))) + try { + return { + trackers: this.prepareHostsForQuery(normalized.slice(0, run.limits.maxTrackers), 'SLAP trackers'), + normalized } - return cached.hosts.slice() + } catch (error) { + run.noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) + return { trackers: [], normalized } } + } - // no cache: coalesce concurrent requests - if (this.hostsInFlight.has(service)) { - try { - const hosts = await this.hostsInFlight.get(service) - if (typeof hosts !== 'object') { - throw new TypeError('Hosts is not defined.') - } - return hosts.slice() - } catch { - // fall through to a fresh attempt below + private async lookupSlapTrackerHosts( + run: LookupQueryRun, + tracker: string, + signal: AbortSignal, + charge: (bytes: number) => void + ): Promise { + const answer = await this.lookupHostWithTracking( + tracker, + { service: 'ls_slap', query: { service: run.question.service } }, + MAX_TRACKER_WAIT_TIME, + signal, + { + maxResponseBytes: run.limits.maxResponseBytes, + maxOutputs: run.limits.maxOutputs, + consumeBytes: charge } + ) + const hosts = isOutputListAnswer(answer) ? this.extractHostsFromAnswer(answer, run.question.service) : [] + for (const host of hosts) { + if (this.advertisedBy.size >= this.hostsMaxEntries * run.limits.maxHosts) this.evictOldest(this.advertisedBy) + this.advertisedBy.set(host, tracker) } + return hosts + } - const promise = this.refreshHosts(service).finally(() => { - this.hostsInFlight.delete(service) - }) - this.hostsInFlight.set(service, promise) - const hosts = await promise - return hosts.slice() + private completeLookupDiscoveryRefresh( + run: LookupQueryRun, + key: string, + discovery: LookupDiscovery, + state: LookupDiscoveryUpdate, + abandoned: boolean + ): void { + if (this.hostsInFlight.get(key) !== discovery) return + this.hostsInFlight.delete(key) + if (abandoned) return + const hosts = Array.from(new Set(Array.from(state.sources.values()).flat())).slice(0, run.limits.maxHosts) + this.rememberDiscoveredHosts(run.question.service, hosts, run.limits, state) } - /** - * Actually resolves competent hosts from SLAP trackers and updates cache. - */ - private async refreshHosts( - service: string, - requireAvailable: boolean = false - ): Promise { - const hosts = await this.findCompetentHosts(service, requireAvailable) - const expiresAt = Date.now() + this.hostsTtlMs + private createLookupDiscovery(run: LookupQueryRun, key: string): LookupDiscovery { + const selected = this.selectSlapTrackers(run) + let discovery: LookupDiscovery + discovery = new LookupDiscovery( + selected.trackers, + run.limits, + async (tracker, signal, charge) => await this.lookupSlapTrackerHosts(run, tracker, signal, charge), + (state, abandoned) => this.completeLookupDiscoveryRefresh(run, key, discovery, state, abandoned) + ) + if (this.slapTrackers.length > run.limits.maxTrackers) discovery.state.limitsHit.add('maxTrackers') + if ( + selected.normalized.length !== this.slapTrackers.length || + selected.trackers.length < Math.min(selected.normalized.length, run.limits.maxTrackers) + ) { + discovery.state.skippedHosts += this.slapTrackers.length - selected.trackers.length + } + return discovery + } - // bounded cache with simple FIFO eviction - if (!this.hostsCache.has(service) && this.hostsCache.size >= this.hostsMaxEntries) { - const oldestKey = this.hostsCache.keys().next().value - if (oldestKey !== undefined) this.hostsCache.delete(oldestKey) + private refreshLookupDiscovery(run: LookupQueryRun, plan: LookupDiscoveryPlan): void { + run.sourceQuota = Math.max( + 1, + Math.floor( + (run.limits.maxHosts - run.seen.size) / + Math.max(1, Math.min(this.slapTrackers.length, run.limits.maxTrackers)) + ) + ) + let discovery = this.hostsInFlight.get(plan.key) + if (discovery === undefined) { + discovery = this.createLookupDiscovery(run, plan.key) + this.hostsInFlight.set(plan.key, discovery) + } + run.releaseDiscovery = discovery.subscribe(state => this.acceptLookupDiscovery(run, state)) + if (run.controller.signal.aborted) run.releaseDiscovery() + } + + private admitDiscoveredLookupSources(run: LookupQueryRun): void { + const plan = this.planLookupDiscovery(run) + if (plan.cached !== undefined && plan.cacheHasAvailableHost) { + this.admitQuotaLimitedLookupHosts(run, 'cache', plan.cached.hosts, plan.initialQuota, 'maxHosts') + } + if (plan.configuredAdditional.length > 0) { + this.admitQuotaLimitedLookupHosts(run, 'additional', plan.configuredAdditional, plan.initialQuota, 'maxHosts') + } + if (plan.refresh) this.refreshLookupDiscovery(run, plan) + else this.reuseCachedLookupDiscovery(run, plan.cached) + } + + private admitLookupSources(run: LookupQueryRun): void { + if ( + run.question.service === 'ls_slap' || + this.hostOverrides[run.question.service] != null || + this.networkPreset === 'local' + ) { + this.admitConfiguredLookupSources(run) + return + } + this.admitDiscoveredLookupSources(run) + } + + private syncLookupDiscoveryProgress(run: LookupQueryRun, state: LookupDiscoveryUpdate): void { + run.session.trackersTotal = state.trackersTotal + run.session.trackersCompleted = state.trackersCompleted + run.session.trackersFailed = state.trackersFailed + run.session.skippedHosts += state.skippedHosts - run.discoverySkipped + run.discoverySkipped = state.skippedHosts + for (const name of state.limitsHit) run.session.limit(name) + } + + private chargeLookupDiscoveryBytes(run: LookupQueryRun, state: LookupDiscoveryUpdate): boolean { + try { + this.consumeLookupQueryBytes(run, state.receivedBytes - run.discoveryBytes) + } catch (error) { + if (error instanceof LookupResourceLimitError) run.session.limit(error.limit) + run.controller.abort() + run.releaseDiscovery?.() + run.queue.cancel() + run.discoveryFinished = true + return false + } + run.discoveryBytes = state.receivedBytes + return true + } + + private admitLookupDiscoverySources(run: LookupQueryRun, state: LookupDiscoveryUpdate): void { + for (const [source, hosts] of state.sources) { + if (run.processedSources.has(source)) continue + run.processedSources.add(source) + this.admitQuotaLimitedLookupHosts(run, source, hosts, run.sourceQuota, 'maxHostsPerTracker') + } + } + + private acceptLookupDiscovery(run: LookupQueryRun, state: LookupDiscoveryUpdate): void { + if (run.controller.signal.aborted) return + this.syncLookupDiscoveryProgress(run, state) + if (!this.chargeLookupDiscoveryBytes(run, state)) return + this.admitLookupDiscoverySources(run, state) + run.session.discoveryComplete = + state.done && + state.trackersFailed === 0 && + state.limitsHit.size === 0 && + state.skippedHosts === 0 + run.session.wake() + if (state.done) this.finishLookupSources(run) + } + + private beginLookupQueryRun(run: LookupQueryRun): void { + this.telemetry.capture({ + name: 'sdk.overlay.lookup.started', + component: 'sdk.lookup-resolver', + severity: 'debug', + correlationId: run.session.correlationId, + attributes: { service: run.question.service, network: this.networkPreset, hostCount: 0 } + }) + if (run.options?.signal?.aborted === true || run.iteratorSignal.aborted) { + this.stopLookupQueryRun(run, 'cancelled') + } + if (!run.controller.signal.aborted) this.admitLookupSources(run) + else run.queue.cancel() + void run.queue.done.then(() => this.finishLookupQueryRun(run)) + } + + private async *queryProgress( + inputQuestion: LookupQuestion, + timeout: number | undefined, + options: LookupQueryOptions | undefined, + iteratorSignal: AbortSignal + ): AsyncIterable { + // Capture JSON wire values once, before any discovery or queued host can + // observe a caller's later mutation. Custom non-JSON questions retain their + // historical facilitator-defined semantics when they cannot be cloned. + const question = this.cloneLookupQuestion(inputQuestion) + const limits = this.lookupQueryLimits(options) + const deadlineMs = options?.deadlineMs ?? 10_000 + this.assertLookupDeadline(deadlineMs) + if (this.activeQueries >= 128) throw new LookupResourceLimitError('activeQueries') + this.activeQueries++ + const run = this.createLookupQueryRun(question, timeout, options, limits, iteratorSignal, deadlineMs) + try { + this.beginLookupQueryRun(run) + for await (const progress of run.session.progress()) { + if (progress.isFinal) { + this.captureLookupCompletedTelemetry(question.service, progress, Date.now() - run.session.startedAt) + this.cleanupLookupQueryRun(run) + } + yield progress + } + } finally { + this.cleanupLookupQueryRun(run) } - this.hostsCache.set(service, { hosts, expiresAt }) - return hosts } /** @@ -1283,70 +1773,6 @@ export default class LookupResolver { return hosts } - /** - * Returns a list of competent hosts for a given lookup service. - * Resolves as soon as the first SLAP tracker responds with valid hosts. - * Remaining trackers continue in the background for reputation tracking. - * @param service Service for which competent hosts are to be returned - * @returns Array of hosts competent for resolving queries - */ - private async findCompetentHosts( - service: string, - requireAvailable: boolean = false - ): Promise { - const query: LookupQuestion = { - service: 'ls_slap', - query: { service } - } - - const trackerHosts = this.prepareHostsForQuery(this.slapTrackers, 'SLAP trackers') - if (trackerHosts.length === 0) return [] - - // Fire all trackers, resolve as soon as any returns valid hosts. - // Remaining trackers continue in the background for reputation tracking. - return await new Promise(resolve => { - const allHosts = new Set() - let resolved = false - let pending = trackerHosts.length - - for (const tracker of trackerHosts) { - this.lookupHostWithTracking(tracker, query, MAX_TRACKER_WAIT_TIME) - .then(answer => { - const hosts = isOutputListAnswer(answer) - ? this.extractHostsFromAnswer(answer, service) - : [] - for (const h of hosts) { - if (!allHosts.has(h)) { - allHosts.add(h) - // First-seen attribution: the tracker that surfaced this host - // gets credit, used by onUnreachableHost callbacks. - this.advertisedBy.set(h, tracker) - } - } - const now = Date.now() - const foundAvailable = [...allHosts].some(host => { - const backoffUntil = this.hostReputation.snapshot(host)?.backoffUntil ?? 0 - return backoffUntil <= now - }) - if (!resolved && allHosts.size > 0 && (!requireAvailable || foundAvailable)) { - resolved = true - resolve([...allHosts]) - } - }) - .catch(() => { - /* tracker failure tracked in reputation */ - }) - .finally(() => { - pending-- - if (pending === 0 && !resolved) { - resolved = true - resolve([...allHosts]) - } - }) - } - }) - } - /** * Resolve a txid for an aggregated lookup output. Uses the threaded-through `output.txid` * fast path when present; otherwise memoizes Transaction.fromBEEF(beef).id('hex') keyed by @@ -1359,7 +1785,7 @@ export default class LookupResolver { if (typeof output.txid === 'string' && output.txid.length > 0) { return output.txid } - const keyForBeef = Array.isArray(output.beef) ? output.beef.join(',') : '' + const keyForBeef = Utils.toHex(sha256(output.beef)) const memo = this.txMemo.get(keyForBeef) if (typeof memo === 'object' && memo !== null && memo.expiresAt > now) { return memo.txId @@ -1380,6 +1806,41 @@ export default class LookupResolver { if (firstKey !== undefined) m.delete(firstKey) } + /** + * Remember SLAP hosts for a service. A tighter-limit discovery must not + * replace a still-fresh broader cache, and a later broader query must not + * treat a truncated entry as complete. + */ + private rememberDiscoveredHosts( + service: string, + hosts: string[], + limits: LookupLimits, + state: LookupDiscoveryUpdate + ): void { + const existing = this.hostsCache.get(service) + const now = Date.now() + if ( + existing !== undefined && + existing.expiresAt > now && + lookupDiscoveryCovers(existing, limits) && + !lookupDiscoveryCovers(lookupDiscoveryBound(limits), existing) + ) { + return + } + if (existing === undefined && this.hostsCache.size >= this.hostsMaxEntries) { + this.evictOldest(this.hostsCache) + } + this.hostsCache.set(service, { + ...lookupDiscoveryBound(limits), + hosts, + expiresAt: now + this.hostsTtlMs, + discoveryComplete: + state.trackersFailed === 0 && state.limitsHit.size === 0 && state.skippedHosts === 0, + trackersFailed: state.trackersFailed, + limitsHit: Array.from(state.limitsHit) + }) + } + private assertValidOverrideServices(overrides: Record): void { for (const service of Object.keys(overrides)) { if (!service.startsWith('ls_')) { @@ -1402,52 +1863,140 @@ export default class LookupResolver { ) } - private async lookupHostWithTracking( - host: string, - question: LookupQuestion, - timeout?: number - ): Promise { - const startedAt = Date.now() - const effectiveTimeout = - typeof timeout === 'number' && Number.isFinite(timeout) && timeout >= 0 - ? timeout - : DEFAULT_LOOKUP_TIMEOUT - const deadline = createDeadline(effectiveTimeout) + private effectiveLookupTimeout(timeout: number | undefined): number { + if (typeof timeout === 'number' && Number.isFinite(timeout) && timeout >= 0) return timeout + return DEFAULT_LOOKUP_TIMEOUT + } + + private startTrackedLookup(args: { + host: string + question: LookupQuestion + timeout: number | undefined + signal: AbortSignal | undefined + controller: AbortController + options: LookupRequestOptions | undefined + reported: { bytes: number } + }): Promise { + const { host, question, timeout, signal, controller, options, reported } = args + const requestOptions = { + ...options, + consumeBytes: (bytes: number): void => { + options?.consumeBytes?.(bytes) + reported.bytes += bytes + } + } // Start the custom facilitator in a promise chain so synchronous throws // become rejections governed by the same wall-clock deadline. - const lookupPromise = Promise.resolve().then(() => - this.facilitator.lookup(host, question, timeout) - ) + const lookupPromise = Promise.resolve().then(() => { + if (signal?.aborted === true) throw lookupAbortError() + return this.facilitator.lookup(host, question, timeout, controller.signal, requestOptions) + }) lookupPromise.catch(() => { /* deadline may win while custom facilitator settles later */ }) + return lookupPromise + } - let answer: LookupFacilitatorAnswer - try { - answer = await Promise.race([lookupPromise, deadline.promise]) - } catch (err) { - const normalized = normalizeLookupError(err, deadline.didTimeOut()) - if (!isSemanticLookupRejection(err)) this.hostReputation.recordFailure(host, normalized) - throw isSemanticLookupRejection(err) ? err : normalized - } finally { - deadline.cancel() + private assertTrackedOutputBudget( + answer: LookupAnswer, + options: LookupRequestOptions | undefined, + reportedBytes: number + ): void { + let bytes = 0 + for (const output of answer.outputs) { + bytes += output.beef.length + (output.context?.length ?? 0) + if (bytes > (options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes)) { + throw new LookupResourceLimitError('maxResponseBytes') + } } + if (reportedBytes === 0) options?.consumeBytes?.(bytes) + } + private assertTrackedLookupAnswer( + answer: LookupFacilitatorAnswer, + options: LookupRequestOptions | undefined, + reportedBytes: number + ): void { + if (answer?.type !== 'output-list' || !Array.isArray(answer.outputs)) return + if (answer.outputs.length > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) { + throw new LookupResourceLimitError('maxOutputs') + } + if (!isOutputListAnswer(answer)) return + this.assertTrackedOutputBudget(answer, options, reportedBytes) + } + + private completeTrackedLookup( + host: string, + answer: LookupFacilitatorAnswer, + startedAt: number, + reportedBytes: number, + options: LookupRequestOptions | undefined, + signal: AbortSignal | undefined + ): LookupFacilitatorAnswer { + if (signal?.aborted === true) throw lookupAbortError() + this.assertTrackedLookupAnswer(answer, options, reportedBytes) if (isOutputListAnswer(answer)) { this.hostReputation.recordSuccess(host, Date.now() - startedAt) return answer } - // A valid freeform response is neutral: it proves this request reached the // service, but it must not erase an availability backoff established by a // concurrent failing request and cannot contribute to output aggregation. if (isFreeformAnswer(answer)) return answer - const malformed = new Error('Malformed lookup response') this.hostReputation.recordFailure(host, malformed) throw malformed } + private throwTrackedLookupFailure( + host: string, + err: unknown, + signal: AbortSignal | undefined, + deadline: Deadline + ): never { + if (signal?.aborted === true) throw lookupAbortError() + if (err instanceof LookupResourceLimitError) throw err + if (isSemanticLookupRejection(err)) throw err + const normalized = normalizeLookupError(err, deadline.didTimeOut()) + this.hostReputation.recordFailure(host, normalized) + throw normalized + } + + private async lookupHostWithTracking( + host: string, + question: LookupQuestion, + timeout?: number, + signal?: AbortSignal, + options?: LookupRequestOptions + ): Promise { + const startedAt = Date.now() + const controller = new AbortController() + const abort = (): void => controller.abort() + signal?.addEventListener('abort', abort, { once: true }) + const deadline = createDeadline(this.effectiveLookupTimeout(timeout), controller) + const reported = { bytes: 0 } + const lookupPromise = this.startTrackedLookup({ + host, + question, + timeout, + signal, + controller, + options, + reported + }) + let answer: LookupFacilitatorAnswer + try { + answer = await withLookupAbort(Promise.race([lookupPromise, deadline.promise]), signal) + } catch (err) { + this.throwTrackedLookupFailure(host, err, signal, deadline) + } finally { + deadline.cancel() + signal?.removeEventListener('abort', abort) + controller.abort() + } + return this.completeTrackedLookup(host, answer, startedAt, reported.bytes, options, signal) + } + private captureHostTelemetry( service: string, host: string, diff --git a/packages/sdk/src/overlay-tools/LookupResources.ts b/packages/sdk/src/overlay-tools/LookupResources.ts new file mode 100644 index 000000000..8fee3a773 --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupResources.ts @@ -0,0 +1,83 @@ +/** Operational client limits, not BEEF validity or service authority rules. */ +export interface LookupLimits { + maxHosts: number + maxHostsPerTracker: number + maxTrackers: number + hostConcurrency: number + trackerConcurrency: number + maxResponseBytes: number + maxTotalBytes: number + maxOutputs: number + maxEvidenceOutputs: number + maxEvidenceBytes: number +} + +/** Finite defaults; applications with larger proofs can raise these explicitly. */ +export const DEFAULT_LOOKUP_LIMITS: Readonly = Object.freeze({ + maxHosts: 256, + maxHostsPerTracker: 64, + maxTrackers: 16, + hostConcurrency: 8, + trackerConcurrency: 4, + maxResponseBytes: 32 * 1024 * 1024, + maxTotalBytes: 64 * 1024 * 1024, + maxOutputs: 4096, + maxEvidenceOutputs: 512, + maxEvidenceBytes: 16 * 1024 * 1024 +}) + +export class LookupResourceLimitError extends Error { + constructor(readonly limit: string) { + super(`Lookup resource limit reached: ${limit}`) + this.name = 'LookupResourceLimitError' + } +} + +export function lookupLimits(...overrides: Array | undefined>): LookupLimits { + const limits = Object.assign({}, DEFAULT_LOOKUP_LIMITS, ...overrides) + for (const [name, value] of Object.entries(limits)) { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + throw new RangeError(`Lookup limit ${name} must be a positive safe integer`) + } + } + return limits +} + +/** Preserve distinct paths and ports; remove only a final slash and URL fragments. */ +export function normalizeLookupHost(host: string, allowParameters: boolean = false): string | null { + if (typeof host !== 'string' || host.length > 2048) return null + try { + const url = new URL(host) + if (!['https:', 'http:'].includes(url.protocol) || url.username !== '' || url.password !== '') return null + // A query/fragment has no defined meaning before the /lookup route suffix. + if (!allowParameters && (url.search !== '' || url.hash !== '')) return null + return url.href.replace(/\/$/, '') + } catch { + return null + } +} + +export function lookupAbortError(): Error { + const error = new Error('Lookup cancelled') + error.name = 'AbortError' + return error +} + +/** A non-cooperative transport cannot retain a cancelled waiter. */ +export async function withLookupAbort(work: Promise, signal?: AbortSignal): Promise { + if (signal === undefined) return await work + if (signal.aborted) { + void work.catch(() => {}) + throw lookupAbortError() + } + let abort = (): void => {} + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(lookupAbortError()) + signal.addEventListener('abort', abort, { once: true }) + }) + try { + return await Promise.race([work, cancelled]) + } finally { + signal.removeEventListener('abort', abort) + } +} diff --git a/packages/sdk/src/overlay-tools/LookupResponseReader.ts b/packages/sdk/src/overlay-tools/LookupResponseReader.ts new file mode 100644 index 000000000..7ef119add --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupResponseReader.ts @@ -0,0 +1,212 @@ +import { LookupResourceLimitError } from './LookupResources.js' + +/** Options controlling a bounded lookup response read. */ +export interface LookupResponseReaderOptions { + /** Cancels a pending stream read when the lookup request is aborted. */ + signal?: AbortSignal + /** Maximum number of response bytes to retain. */ + maxResponseBytes: number + /** Charges accepted bytes to the caller's aggregate response budget. */ + consumeBytes?: (bytes: number) => void +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('The operation was aborted.', 'AbortError') +} + +function assertValidMaximum(maxResponseBytes: number): void { + if (!Number.isSafeInteger(maxResponseBytes) || maxResponseBytes < 0) { + throw new RangeError('maxResponseBytes must be a non-negative safe integer') + } +} + +function assertDeclaredLengthIsWithinLimit(response: Response, maxResponseBytes: number): void { + const contentLength = response.headers.get('content-length') + if (contentLength === null) return + + const normalized = contentLength.trim() + // Content-Length is decimal bytes. Treat malformed fields as unknown rather + // than accidentally accepting a notation such as "1e6". + if (!/^\d+$/.test(normalized)) return + + const declaredLength = Number(normalized) + if (!Number.isSafeInteger(declaredLength) || declaredLength > maxResponseBytes) { + throw new LookupResourceLimitError('maxResponseBytes') + } +} + +async function readWithAbort( + reader: ReadableStreamDefaultReader, + signal: AbortSignal | undefined +): Promise> { + if (signal === undefined) return await reader.read() + if (signal.aborted) throw abortReason(signal) + + return await new Promise>((resolve, reject) => { + let settled = false + const finish = (callback: () => void): void => { + if (settled) return + settled = true + signal.removeEventListener('abort', onAbort) + callback() + } + const onAbort = (): void => finish(() => reject(abortReason(signal))) + + signal.addEventListener('abort', onAbort, { once: true }) + Promise.resolve() + .then(() => reader.read()) + .then( + result => finish(() => resolve(result)), + error => finish(() => reject(error)) + ) + + // Do not miss an abort that happened while registering the listener. + if (signal.aborted) onAbort() + }) +} + +function cleanUpFailedRead(reader: ReadableStreamDefaultReader, reason: unknown): void { + Promise.resolve() + .then(() => reader.cancel(reason)) + .catch(() => undefined) + + try { + reader.releaseLock() + } catch { + // The lock may already have been released by a nonstandard stream. + } +} + +function expandedBuffer( + buffer: Uint8Array, + requiredLength: number, + maxResponseBytes: number +): Uint8Array { + if (requiredLength <= buffer.byteLength) return buffer + + const initialCapacity = Math.min(maxResponseBytes, 1024) + const doubledCapacity = Math.min(maxResponseBytes, buffer.byteLength * 2) + const capacity = Math.max( + requiredLength, + buffer.byteLength === 0 ? initialCapacity : doubledCapacity + ) + const expanded = new Uint8Array(capacity) + expanded.set(buffer) + return expanded +} + +function yieldToEventLoop(): Promise { + return new Promise(resolve => setTimeout(resolve, 0)) +} + +async function yieldAfterReadIfNeeded( + readOperations: number, + signal: AbortSignal | undefined +): Promise { + if (readOperations % 64 !== 0) return + await yieldToEventLoop() + if (signal?.aborted) throw abortReason(signal) +} + +async function accumulateLookupResponseChunk( + bytes: Uint8Array, + totalLength: number, + value: Uint8Array, + readOperations: number, + options: LookupResponseReaderOptions +): Promise<{ bytes: Uint8Array, totalLength: number }> { + const { signal, maxResponseBytes, consumeBytes } = options + if (value.byteLength === 0) { + // An eagerly fulfilled read() still schedules only microtasks. Yielding + // periodically lets timers deliver cancellation for endless empty input. + await yieldAfterReadIfNeeded(readOperations, signal) + return { bytes, totalLength } + } + + if (value.byteLength > maxResponseBytes - totalLength) { + throw new LookupResourceLimitError('maxResponseBytes') + } + + consumeBytes?.(value.byteLength) + const nextLength = totalLength + value.byteLength + const expanded = expandedBuffer(bytes, nextLength, maxResponseBytes) + // Streams are allowed to reuse a producer-owned Uint8Array. Copy each + // accepted chunk now instead of retaining a mutable producer reference. + expanded.set(value, totalLength) + // Copy before yielding: a producer may reuse or mutate its buffer while + // the task queue runs. + await yieldAfterReadIfNeeded(readOperations, signal) + return { bytes: expanded, totalLength: nextLength } +} + +function releaseLookupResponseReader(reader: ReadableStreamDefaultReader): void { + try { + reader.releaseLock() + } catch { + // A nonstandard stream may have released its lock itself. + } +} + +async function readLookupResponseStream( + reader: ReadableStreamDefaultReader, + response: Response, + options: LookupResponseReaderOptions +): Promise { + const { signal } = options + let succeeded = false + let failure: unknown + try { + assertDeclaredLengthIsWithinLimit(response, options.maxResponseBytes) + if (signal?.aborted === true) throw abortReason(signal) + + let bytes: Uint8Array = new Uint8Array(0) + let totalLength = 0 + let readOperations = 0 + while (true) { + const { done, value } = await readWithAbort(reader, signal) + readOperations++ + if (done) break + const next = await accumulateLookupResponseChunk( + bytes, + totalLength, + value ?? new Uint8Array(0), + readOperations, + options + ) + bytes = next.bytes + totalLength = next.totalLength + } + + succeeded = true + return bytes.subarray(0, totalLength) + } catch (error) { + failure = error + throw error + } finally { + if (succeeded) releaseLookupResponseReader(reader) + else cleanUpFailedRead(reader, failure) + } +} + +/** + * Reads a lookup response incrementally while enforcing a per-response bound. + * + * This deliberately does not use Response.text(), json(), or arrayBuffer(), + * because those APIs buffer the complete body before a limit can be enforced. + */ +export async function readLookupResponseBytes( + response: Response, + options: LookupResponseReaderOptions +): Promise { + const { signal, maxResponseBytes } = options + assertValidMaximum(maxResponseBytes) + + const body = response.body + if (body === null) { + assertDeclaredLengthIsWithinLimit(response, maxResponseBytes) + if (signal?.aborted === true) throw abortReason(signal) + return new Uint8Array(0) + } + + return await readLookupResponseStream(body.getReader(), response, options) +} diff --git a/packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts new file mode 100644 index 000000000..c63acdc52 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts @@ -0,0 +1,146 @@ +import { LookupDiscovery } from '../LookupDiscovery.js' +import { LookupResourceLimitError, lookupLimits } from '../LookupResources.js' + +const limits = lookupLimits({ + maxHosts: 4, + maxHostsPerTracker: 2, + maxTrackers: 4, + hostConcurrency: 2, + trackerConcurrency: 1, + maxResponseBytes: 100, + maxTotalBytes: 20, + maxOutputs: 8, + maxEvidenceOutputs: 8, + maxEvidenceBytes: 100 +}) + +function deferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolve!: (value: T) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +describe('LookupDiscovery', () => { + it('rejects further byte charges after the last subscriber abandons discovery', async () => { + const started = deferred<(bytes: number) => void>() + const hang = deferred() + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async (_tracker, _signal, consume) => { + started.resolve(consume) + return await hang.promise + }, + () => {} + ) + const unsubscribe = discovery.subscribe(() => {}) + const consume = await started.promise + unsubscribe() + + expect(() => consume(1)).toThrow('Lookup resource limit reached: abandoned') + + hang.resolve([]) + }) + + it('charges tracker bytes until maxTotalBytes then records the limit without a tracker failure', async () => { + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async (_tracker, _signal, consume) => { + consume(10) + consume(11) + return [] + }, + () => {} + ) + const finished = deferred() + discovery.subscribe(state => { + if (state.done) finished.resolve() + }) + await finished.promise + + expect(discovery.state.receivedBytes).toBe(10) + expect(discovery.state.limitsHit.has('maxTotalBytes')).toBe(true) + expect(discovery.state.trackersFailed).toBe(0) + expect(discovery.state.trackersCompleted).toBe(1) + }) + + it('skips invalid, duplicate, and over-share hosts from one tracker', async () => { + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async () => [ + 'not-a-url', + 'ftp://blocked.example', + 'https://user:pass@secret.example', + 'https://host.example', + 'https://host.example/', + 'https://second.example', + 'https://third.example' + ], + () => {} + ) + const finished = deferred() + discovery.subscribe(state => { + if (state.done) finished.resolve() + }) + await finished.promise + + expect(discovery.state.sources.get('https://tracker.example')).toEqual([ + 'https://host.example', + 'https://second.example' + ]) + expect(discovery.state.skippedHosts).toBeGreaterThanOrEqual(3) + expect(discovery.state.limitsHit.has('maxHostsPerTracker')).toBe(true) + }) + + it('records a resource-limit error from lookup without counting a tracker failure', async () => { + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async () => { + throw new LookupResourceLimitError('maxResponseBytes') + }, + () => {} + ) + const finished = deferred() + discovery.subscribe(state => { + if (state.done) finished.resolve() + }) + await finished.promise + + expect(discovery.state.limitsHit.has('maxResponseBytes')).toBe(true) + expect(discovery.state.trackersFailed).toBe(0) + }) + + it('does not record hosts after the last subscriber abandons an in-flight tracker', async () => { + const started = deferred() + const release = deferred() + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async () => { + started.resolve() + return await release.promise + }, + () => {} + ) + const unsubscribe = discovery.subscribe(() => {}) + await started.promise + unsubscribe() + release.resolve(['https://late-host.example']) + while (discovery.state.trackersCompleted === 0) { + await new Promise(resolve => setImmediate(resolve)) + } + + expect(discovery.state.sources.size).toBe(0) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts new file mode 100644 index 000000000..321b275fa --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts @@ -0,0 +1,62 @@ +import { LookupHostQueue } from '../LookupHostQueue.js' + +describe('LookupHostQueue', () => { + it('does not enqueue hosts after sources close or the queue is cancelled', async () => { + const ran: string[] = [] + const closed = new LookupHostQueue( + 8, + 1, + async host => { + ran.push(host) + }, + () => {} + ) + closed.finishSources() + closed.add('late', ['https://late.example']) + await closed.done + + const cancelled = new LookupHostQueue( + 8, + 1, + async host => { + ran.push(host) + }, + () => {} + ) + cancelled.cancel() + cancelled.add('late', ['https://cancelled.example']) + await cancelled.done + + expect(ran).toEqual([]) + }) + + it('ignores duplicate hosts and reports overflow past maxHosts', async () => { + const ran: string[] = [] + const skipped: Array<[number, boolean]> = [] + const queue = new LookupHostQueue( + 2, + 2, + async host => { + ran.push(host) + }, + (count, limited) => { + skipped.push([count, limited]) + } + ) + queue.add('tracker-a', [ + 'https://a.example', + 'https://a.example', + 'https://b.example', + 'https://c.example' + ]) + queue.add('tracker-b', ['https://a.example', 'https://d.example']) + queue.finishSources() + await queue.done + + expect(ran).toEqual(['https://a.example', 'https://b.example']) + expect(skipped).toEqual([ + [1, true], + [1, true] + ]) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts index 02aff25a4..3a782138a 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts @@ -15,6 +15,15 @@ const mockFacilitator = { lookup: jest.fn() } +const jsonResponse = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }) + +const octetResponse = (payload: Uint8Array): Response => + new Response(payload, { headers: { 'content-type': 'application/octet-stream' } }) + // -------------------------------------------------------------------------- // Sample BEEFs for use in tests // -------------------------------------------------------------------------- @@ -469,11 +478,9 @@ describe('LookupResolver – additional coverage', () => { }) it('allows HTTP URLs when allowHTTP is true', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'output-list', outputs: [] }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('http://localhost:8080', { service: 'ls_test', @@ -482,13 +489,63 @@ describe('LookupResolver – additional coverage', () => { expect(result).toEqual({ type: 'output-list', outputs: [] }) }) - it('handles HTTP error responses by throwing', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: false, - status: 503, - headers: { get: () => 'application/json' }, - json: async () => ({}) + it('refuses to follow a redirect away from the advertised lookup host', async () => { + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) + const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, false) + await facilitator.lookup('https://advertised.example', { service: 'ls_test', query: {} }) + expect(mockFetch).toHaveBeenCalledWith( + 'https://advertised.example/lookup', + expect.objectContaining({ redirect: 'error' }) + ) + }) + + it('treats a rejected redirect as a host failure rather than a crash', async () => { + // fetch rejects with a TypeError when redirect: 'error' meets a 307/308. + const mockFetch = jest.fn().mockRejectedValue(new TypeError('unexpected redirect')) + const resolver = new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(mockFetch, false), + hostOverrides: { ls_redirect: ['https://redirecting.example'] } + }) + + const result = await resolver.queryDetailed({ service: 'ls_redirect', query: {} }) + + expect(result.answer).toEqual({ type: 'output-list', outputs: [] }) + expect(result.progress).toMatchObject({ + hostCount: 1, + failedHosts: 1, + successfulHosts: 0, + rejectedHosts: 0, + terminalReason: 'settled' + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(mockFetch).toHaveBeenCalledWith( + 'https://redirecting.example/lookup', + expect.objectContaining({ redirect: 'error' }) + ) + }) + + it('refuses to follow a redirect on SLAP tracker discovery requests', async () => { + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) + const resolver = new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(mockFetch, false), + slapTrackers: ['https://tracker.example'] }) + + await expect(resolver.query({ service: 'ls_redirect_tracker', query: {} })).rejects.toThrow( + 'No competent mainnet hosts found' + ) + expect(mockFetch).toHaveBeenCalledWith( + 'https://tracker.example/lookup', + expect.objectContaining({ redirect: 'error' }) + ) + }) + + it('handles HTTP error responses by throwing', async () => { + const mockFetch = jest.fn().mockResolvedValue(jsonResponse({}, 503)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await expect( facilitator.lookup('http://host', { service: 'ls_test', query: {} }) @@ -544,12 +601,7 @@ describe('LookupResolver – additional coverage', () => { const beefBuf = Buffer.from(beef) const payload = Buffer.concat([nOutpoints, txid, outputIndex, contextLen, beefBuf]) - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/octet-stream' }, - arrayBuffer: async () => - payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength) - }) + const mockFetch = jest.fn().mockResolvedValue(octetResponse(payload)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('http://host', { service: 'ls_test', query: {} }) @@ -581,12 +633,9 @@ describe('LookupResolver – additional coverage', () => { 'Application/Octet-Stream', ' application/octet-stream ' ]) { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => header }, - arrayBuffer: async () => - payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength) - }) + const mockFetch = jest + .fn() + .mockResolvedValue(new Response(payload, { headers: { 'content-type': header } })) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('https://host', { service: 'ls_test', query: {} }) expect(result.type).toBe('output-list') @@ -620,12 +669,7 @@ describe('LookupResolver – additional coverage', () => { beefBuf ]) - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/octet-stream' }, - arrayBuffer: async () => - payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength) - }) + const mockFetch = jest.fn().mockResolvedValue(octetResponse(payload)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('http://host', { service: 'ls_test', query: {} }) @@ -688,11 +732,9 @@ describe('LookupResolver – additional coverage', () => { }) it('sends correct request body to /lookup endpoint', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'output-list', outputs: [] }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const question = { service: 'ls_test', query: { filter: 'abc' } } await facilitator.lookup('http://host', question) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts new file mode 100644 index 000000000..f90b61755 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -0,0 +1,1258 @@ +import LookupResolver, { + HTTPSOverlayLookupFacilitator, + LookupAnswerProgress, + LookupResourceLimitError +} from '../LookupResolver' +import { getOverlayHostReputationTracker } from '../HostReputationTracker' +import OverlayAdminTokenTemplate from '../OverlayAdminTokenTemplate' +import { CompletedProtoWallet } from '../../auth/certificates/__tests/CompletedProtoWallet' +import { PrivateKey } from '../../primitives/index' +import { LockingScript } from '../../script/index' +import { Transaction } from '../../transaction/index' + +const makeBeef = (satoshis: number): number[] => + new Transaction(1, [], [{ lockingScript: LockingScript.fromHex('88'), satoshis }], 0).toBEEF() + +const later = async (ms: number): Promise => { + await new Promise(resolve => setTimeout(resolve, ms)) +} + +async function overlayReceipt( + protocol: 'SHIP' | 'SLAP', + scalar: number, + domain: string, + topicOrService: string +): Promise<{ beef: number[]; outputIndex: number }> { + const wallet = new CompletedProtoWallet(new PrivateKey(scalar)) + const token = new OverlayAdminTokenTemplate(wallet) + const lockingScript = await token.lock(protocol, domain, topicOrService) + const transaction = new Transaction(1, [], [{ lockingScript, satoshis: 1 }], 0) + return { beef: transaction.toBEEF(), outputIndex: 0 } +} + +async function slapReceipt( + scalar: number, + domain: string, + service: string +): Promise<{ beef: number[]; outputIndex: number }> { + return await overlayReceipt('SLAP', scalar, domain, service) +} + +describe('LookupResolver dynamic discovery', () => { + beforeEach(() => { + getOverlayHostReputationTracker().reset() + jest.useFakeTimers() + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it('starts a discovered host without waiting for a slower tracker and merges only the late host contribution', async () => { + const fastTracker = 'https://fast-tracker.example' + const slowTracker = 'https://slow-tracker.example' + const fastHost = 'https://fast-host.example' + const lateHost = 'https://late-host.example' + const service = 'ls_dynamic' + const fastReceipt = await slapReceipt(101, fastHost, service) + const lateReceipt = await slapReceipt(102, lateHost, service) + const fastBeef = makeBeef(1) + const lateBeef = makeBeef(2) + const calls: Array<{ url: string; service: string }> = [] + const evidence: Array<{ type: string; host?: string }> = [] + + const lookup = jest.fn(async (url: string, question: { service: string }) => { + calls.push({ url, service: question.service }) + if (url === fastTracker) { + await later(10) + return { type: 'output-list' as const, outputs: [fastReceipt] } + } + if (url === slowTracker) { + await later(100) + return { type: 'output-list' as const, outputs: [lateReceipt] } + } + if (url === fastHost) { + await later(1) + return { type: 'output-list' as const, outputs: [{ beef: fastBeef, outputIndex: 0 }] } + } + if (url === lateHost) { + await later(1) + return { type: 'output-list' as const, outputs: [{ beef: lateBeef, outputIndex: 1 }] } + } + throw new Error(`unexpected host ${url}`) + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + slapTrackers: [fastTracker, slowTracker] + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service, query: { q: 1 } }, undefined, { + graceMs: 0, + onEvidence: event => evidence.push(event) + })) { + progress.push({ ...item, outputs: item.outputs.slice() }) + } + })() + + await jest.advanceTimersByTimeAsync(20) + expect(calls).toContainEqual({ url: fastHost, service }) + expect(calls).not.toContainEqual({ url: lateHost, service }) + expect(calls.filter(call => call.url === fastTracker || call.url === slowTracker)).toEqual([ + { url: fastTracker, service: 'ls_slap' }, + { url: slowTracker, service: 'ls_slap' } + ]) + expect(evidence).toEqual( + expect.arrayContaining([expect.objectContaining({ type: 'output', host: fastHost })]) + ) + + await jest.advanceTimersByTimeAsync(200) + await pending + + const final = progress.at(-1) + expect(final?.isFinal).toBe(true) + expect(final?.outputs).toEqual([ + { beef: fastBeef, outputIndex: 0 }, + { beef: lateBeef, outputIndex: 1 } + ]) + expect(final).toMatchObject({ discoveryComplete: true, trackersTotal: 2, trackersCompleted: 2 }) + // The SLAP receipt is discovery evidence only; it never becomes a topic result. + expect( + final?.outputs.some( + output => output.beef === fastReceipt.beef || output.beef === lateReceipt.beef + ) + ).toBe(false) + }) + + it('queries only eligible advertised hosts for the requested service', async () => { + const fooTracker = 'https://foo-tracker.example' + const barTracker = 'https://bar-tracker.example' + const shipTracker = 'https://ship-tracker.example' + const offlineTracker = 'https://offline-tracker.example' + const fooHost = 'https://foo-host.example' + const barHost = 'https://bar-host.example' + const shipHost = 'https://ship-host.example' + const service = 'ls_foo' + const fooReceipt = await slapReceipt(201, fooHost, service) + const barReceipt = await slapReceipt(202, barHost, 'ls_bar') + const shipReceipt = await overlayReceipt('SHIP', 203, shipHost, service) + const hostCalls: string[] = [] + const lookup = jest.fn(async (url: string, question: { service: string }) => { + if (url === fooTracker) return { type: 'output-list' as const, outputs: [fooReceipt] } + if (url === barTracker) return { type: 'output-list' as const, outputs: [barReceipt] } + if (url === shipTracker) return { type: 'output-list' as const, outputs: [shipReceipt] } + if (url === offlineTracker) throw new Error('tracker offline') + hostCalls.push(url) + expect(question.service).toBe(service) + return { type: 'output-list' as const, outputs: [{ beef: makeBeef(9), outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + slapTrackers: [fooTracker, barTracker, shipTracker, offlineTracker] + }) + const pending = resolver.query({ service, query: {} }) + await jest.runAllTimersAsync() + await expect(pending).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: makeBeef(9), outputIndex: 0 }] + }) + + expect(hostCalls).toEqual([fooHost]) + expect(lookup.mock.calls.map(([url]) => url)).toEqual( + expect.arrayContaining([fooTracker, barTracker, shipTracker, offlineTracker]) + ) + }) + + it('delivers a useful host while another peer never finishes, then settles after the 2s host bound', async () => { + const usefulHost = 'https://useful-hang.example' + const hangingHost = 'https://hanging-peer.example' + const usefulBeef = makeBeef(11) + const lookup = jest.fn(async (url: string, _question: unknown, timeout?: number) => { + expect(timeout).toBeUndefined() + if (url === hangingHost) await new Promise(() => {}) + await later(20) + return { type: 'output-list' as const, outputs: [{ beef: usefulBeef, outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_hang: [usefulHost, hangingHost] } + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_hang', query: {} }, undefined, { + graceMs: 0 + })) { + progress.push({ ...item, outputs: item.outputs.slice() }) + } + })() + + await jest.advanceTimersByTimeAsync(100) + expect(progress.some(item => !item.isFinal && item.outputs.length > 0)).toBe(true) + expect(progress.find(item => item.outputs.length > 0)?.outputs).toEqual([ + { beef: usefulBeef, outputIndex: 0 } + ]) + expect(progress.at(-1)?.isFinal).toBe(false) + + await jest.advanceTimersByTimeAsync(2000) + await pending + expect(progress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + successfulHosts: 1, + failedHosts: 1, + outputs: [{ beef: usefulBeef, outputIndex: 0 }] + }) + }) + + it('unblocks a pending iterator once on abort and reports a cancelled terminal snapshot', async () => { + const host = 'https://pending.example' + const controller = new AbortController() + let requestSignal: AbortSignal | undefined + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + requestSignal = signal + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_abort: [host] } + }) + const iterator = resolver + .query$({ service: 'ls_abort', query: {} }, undefined, { + signal: controller.signal + } as any) + [Symbol.asyncIterator]() + + const first = iterator.next() + await Promise.resolve() + controller.abort(new Error('caller stopped lookup')) + + const terminal = await first + expect(requestSignal?.aborted).toBe(true) + expect(terminal.done).toBe(false) + expect(terminal.value).toMatchObject({ + isFinal: true, + terminalReason: 'cancelled' + }) + await expect(iterator.next()).resolves.toEqual({ done: true, value: undefined }) + }) + + it('rejects query() with an AbortError instead of an empty answer when a queried host is aborted', async () => { + const host = 'https://abort-query.example' + const controller = new AbortController() + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_abort_query: [host] } + }) + const pending = resolver.query({ service: 'ls_abort_query', query: {} }, undefined, { + signal: controller.signal + }) + pending.catch(() => { + /* asserted below */ + }) + + await jest.advanceTimersByTimeAsync(1) + expect(lookup).toHaveBeenCalledTimes(1) + controller.abort(new Error('caller stopped lookup')) + await jest.advanceTimersByTimeAsync(1) + + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'Lookup cancelled' + }) + }) + + it('rejects queryDetailed() with an AbortError when the caller aborts before a host is admitted', async () => { + const controller = new AbortController() + controller.abort(new Error('caller stopped lookup')) + const lookup = jest.fn() + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_abort_early: ['https://abort-early.example'] } + }) + const pending = resolver.queryDetailed( + { service: 'ls_abort_early', query: {} }, + undefined, + { signal: controller.signal } + ) + pending.catch(() => { + /* asserted below */ + }) + + await jest.advanceTimersByTimeAsync(1) + + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'Lookup cancelled' + }) + expect(lookup).not.toHaveBeenCalled() + }) + + it('emits a deadline terminal snapshot when no host receipt arrives', async () => { + const host = 'https://deadline.example' + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_deadline: [host] } + }) + const received: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_deadline', query: {} }, undefined, { + deadlineMs: 25 + } as any)) { + received.push(item) + } + })() + + await jest.advanceTimersByTimeAsync(25) + await pending + + expect(received).toHaveLength(1) + expect(received[0]).toMatchObject({ + isFinal: true, + terminalReason: 'deadline', + outputs: [] + }) + }) + + it('keeps empty, failed, and freeform host receipts distinct at a settled terminal', async () => { + const emptyHost = 'https://empty.example' + const freeformHost = 'https://freeform.example' + const failedHost = 'https://failed.example' + const lookup = jest.fn(async (url: string) => { + if (url === emptyHost) return { type: 'output-list' as const, outputs: [] } + if (url === freeformHost) return { type: 'freeform' as const, result: { supported: false } } + throw new Error('offline') + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_outcomes: [emptyHost, freeformHost, failedHost] } + }) + const values: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_outcomes', query: {} })) + values.push(item) + })() + await jest.runAllTimersAsync() + await pending + + expect(values).toHaveLength(1) + expect(values[0]).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + emptyHosts: 1, + freeformHosts: 1, + failedHosts: 1, + rejectedHosts: 0 + }) + }) + + it('caps evidence delivery independently and reports the limit evidence', async () => { + const host = 'https://bounded.example' + const first = makeBeef(31) + const second = makeBeef(32) + const evidence: Array<{ type: string }> = [] + const resolver = new LookupResolver({ + facilitator: { + lookup: async () => ({ + type: 'output-list', + outputs: [ + { beef: first, outputIndex: 0 }, + { beef: second, outputIndex: 1 } + ] + }) + }, + hostOverrides: { ls_bounded: [host] } + }) + const values: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_bounded', query: {} }, undefined, { + limits: { maxEvidenceOutputs: 1 }, + onEvidence: event => evidence.push(event) + } as any)) { + values.push(item) + } + })() + await jest.runAllTimersAsync() + await pending + + expect(values.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'resource-limit', + limitsHit: expect.arrayContaining(['maxEvidenceOutputs']) + }) + expect(values.at(-1)?.outputs).toHaveLength(2) + expect(evidence).toEqual(expect.arrayContaining([expect.objectContaining({ type: 'limit' })])) + }) + + it('shares one tracker discovery between queries and keeps it alive when one subscriber cancels', async () => { + const tracker = 'https://shared-tracker.example' + const host = 'https://shared-host.example' + const receipt = await slapReceipt(103, host, 'ls_shared') + let completeTracker: (() => void) | undefined + let trackerSignal: AbortSignal | undefined + const lookup = jest.fn( + async (url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => { + if (url === tracker) { + trackerSignal = signal + await new Promise(resolve => { + completeTracker = resolve + }) + return { type: 'output-list' as const, outputs: [receipt] } + } + return { type: 'output-list' as const, outputs: [{ beef: makeBeef(41), outputIndex: 0 }] } + } + ) + const resolver = new LookupResolver({ facilitator: { lookup } as any, slapTrackers: [tracker] }) + const firstAbort = new AbortController() + const first = resolver + .query$({ service: 'ls_shared', query: { caller: 1 } }, undefined, { + signal: firstAbort.signal + } as any) + [Symbol.asyncIterator]() + const secondProgress: LookupAnswerProgress[] = [] + const second = (async () => { + for await (const progress of resolver.query$({ + service: 'ls_shared', + query: { caller: 2 } + })) { + secondProgress.push(progress) + } + })() + + const firstPending = first.next() + await Promise.resolve() + firstAbort.abort() + await expect(firstPending).resolves.toMatchObject({ value: { terminalReason: 'cancelled' } }) + expect(trackerSignal?.aborted).toBe(false) + completeTracker?.() + await jest.runAllTimersAsync() + await second + expect(secondProgress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + outputs: [{ beef: makeBeef(41), outputIndex: 0 }] + }) + expect(lookup.mock.calls.filter(([url]) => url === tracker)).toHaveLength(1) + }) + + it('does not let an abandoned custom-facilitator completion leak into a later query or its evidence', async () => { + const host = 'https://late-custom.example' + const staleBeef = makeBeef(51) + const freshBeef = makeBeef(52) + let resolveStale: + | ((value: { + type: 'output-list' + outputs: Array<{ beef: number[]; outputIndex: number }> + }) => void) + | undefined + let calls = 0 + const lookup = jest.fn(() => { + calls++ + if (calls === 1) { + return new Promise<{ + type: 'output-list' + outputs: Array<{ beef: number[]; outputIndex: number }> + }>(resolve => { + resolveStale = resolve + }) + } + return Promise.resolve({ + type: 'output-list' as const, + outputs: [{ beef: freshBeef, outputIndex: 0 }] + }) + }) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_late_custom: [host] } + }) + const abort = new AbortController() + const abandonedEvidence: Array<{ type: string }> = [] + const abandoned = resolver + .query$({ service: 'ls_late_custom', query: { generation: 1 } }, undefined, { + signal: abort.signal, + onEvidence: event => abandonedEvidence.push(event) + } as any) + [Symbol.asyncIterator]() + const abandonedPending = abandoned.next() + await Promise.resolve() + abort.abort() + await abandonedPending + + const next = resolver.query({ service: 'ls_late_custom', query: { generation: 2 } }) + resolveStale?.({ type: 'output-list', outputs: [{ beef: staleBeef, outputIndex: 0 }] }) + await jest.runAllTimersAsync() + await expect(next).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: freshBeef, outputIndex: 0 }] + }) + expect(abandonedEvidence).toEqual([{ type: 'limit' }]) + }) + + it('stops evidence and aggregate intake when the first evidence callback aborts', async () => { + const host = 'https://callback-abort.example' + const controller = new AbortController() + const events: Array<{ type: string; output?: unknown }> = [] + const resolver = new LookupResolver({ + facilitator: { + lookup: async () => ({ + type: 'output-list' as const, + outputs: [ + { beef: makeBeef(201), outputIndex: 0 }, + { beef: makeBeef(202), outputIndex: 1 }, + { beef: makeBeef(203), outputIndex: 2 } + ] + }) + }, + hostOverrides: { ls_callback_abort: [host] } + }) + const iterator = resolver + .query$({ service: 'ls_callback_abort', query: {} }, undefined, { + signal: controller.signal, + onEvidence: event => { + events.push(event) + if (event.type === 'output') controller.abort() + } + }) + [Symbol.asyncIterator]() + + const terminal = await iterator.next() + expect(terminal.value).toMatchObject({ + isFinal: true, + terminalReason: 'cancelled', + outputs: [] + }) + expect(events.filter(event => event.type === 'output')).toHaveLength(1) + expect(events.filter(event => event.type === 'limit')).toHaveLength(1) + await expect(iterator.next()).resolves.toEqual({ done: true, value: undefined }) + }) + + it('aborts and settles a pending iterator when return() is called', async () => { + const host = 'https://return-pending.example' + let requestSignal: AbortSignal | undefined + const resolver = new LookupResolver({ + facilitator: { + lookup: async (_url, _question, _timeout, signal) => + await new Promise((_resolve, reject) => { + requestSignal = signal + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + }, + hostOverrides: { ls_return_pending: [host] } + }) + const iterator = resolver + .query$({ service: 'ls_return_pending', query: {} }) + [Symbol.asyncIterator]() + const pending = iterator.next() + await Promise.resolve() + await expect(iterator.return?.()).resolves.toEqual({ done: true, value: undefined }) + expect(requestSignal?.aborted).toBe(true) + await expect(pending).resolves.toMatchObject({ + done: false, + value: { isFinal: true, terminalReason: 'cancelled' } + }) + }) + + it('cleans terminal queries before yielding so 130 final-only consumers do not exhaust slots', async () => { + const resolver = new LookupResolver({ + facilitator: { lookup: async () => ({ type: 'output-list' as const, outputs: [] }) }, + hostOverrides: { ls_final_only: ['https://final-only.example'] } + }) + for (let index = 0; index < 130; index++) { + const first = await resolver + .query$({ service: 'ls_final_only', query: { index } }) + [Symbol.asyncIterator]() + .next() + expect(first.value?.isFinal).toBe(true) + } + expect((resolver as any).activeQueries).toBe(0) + }) + + it('enforces decoded BEEF and context retention across hosts independently of reported wire bytes', async () => { + const firstHost = 'https://decoded-one.example' + const secondHost = 'https://decoded-two.example' + const firstOutput = { beef: makeBeef(211), outputIndex: 0, context: Array(10).fill(7) } + const secondOutput = { beef: makeBeef(212), outputIndex: 1, context: Array(10).fill(8) } + const resolver = new LookupResolver({ + facilitator: { + lookup: async (host, _question, _timeout, _signal, requestOptions) => { + requestOptions?.consumeBytes?.(1) + return { + type: 'output-list' as const, + outputs: [host === firstHost ? firstOutput : secondOutput] + } + } + }, + hostOverrides: { ls_decoded_budget: [firstHost, secondHost] } + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$( + { service: 'ls_decoded_budget', query: {} }, + undefined, + { + limits: { + hostConcurrency: 1, + maxTotalBytes: firstOutput.beef.length + firstOutput.context.length + } + } + )) + progress.push(item) + })() + await jest.runAllTimersAsync() + await pending + + expect(progress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'resource-limit', + limitsHit: expect.arrayContaining(['maxTotalBytes']), + receivedBytes: 2, + retainedBytes: firstOutput.beef.length + firstOutput.context.length, + outputs: [firstOutput] + }) + }) + + it('normalizes duplicate configured endpoints and never exceeds the host concurrency budget', async () => { + const one = 'https://one.example' + const two = 'https://two.example' + const three = 'https://three.example' + const calls: string[] = [] + let active = 0 + let peak = 0 + const resolver = new LookupResolver({ + facilitator: { + lookup: async (host: string) => { + calls.push(host) + active++ + peak = Math.max(peak, active) + await later(20) + active-- + return { type: 'output-list' as const, outputs: [] } + } + }, + hostOverrides: { ls_concurrency: [`${one}/`, one, two, three] } + }) + const pending = resolver.query({ service: 'ls_concurrency', query: {} }, undefined, { + limits: { hostConcurrency: 2 } + }) + await jest.runAllTimersAsync() + await pending + + expect(calls).toEqual(expect.arrayContaining([one, two, three])) + expect(calls.filter(host => host === one)).toHaveLength(1) + expect(peak).toBeLessThanOrEqual(2) + }) + + it('gives a later tracker source a turn before draining an earlier tracker flood', async () => { + const firstTracker = 'https://first-tracker.example' + const lateTracker = 'https://late-tracker.example' + const floodHosts = [ + 'https://flood-1.example', + 'https://flood-2.example', + 'https://flood-3.example' + ] + const lateHost = 'https://late-fair.example' + const service = 'ls_fair' + const receipts = await Promise.all([ + ...floodHosts.map((host, index) => slapReceipt(110 + index, host, service)), + slapReceipt(120, lateHost, service) + ]) + const hostCalls: string[] = [] + const resolver = new LookupResolver({ + facilitator: { + lookup: async (host: string) => { + if (host === firstTracker) + return { type: 'output-list' as const, outputs: receipts.slice(0, 3) } + if (host === lateTracker) { + await later(1) + return { type: 'output-list' as const, outputs: [receipts[3]] } + } + hostCalls.push(host) + await later(20) + return { type: 'output-list' as const, outputs: [] } + } + }, + slapTrackers: [firstTracker, lateTracker] + }) + const pending = resolver.query({ service, query: {} }, undefined, { + limits: { hostConcurrency: 1 } + }) + await jest.runAllTimersAsync() + await pending + + expect(hostCalls).toContain(lateHost) + expect(hostCalls.indexOf(lateHost)).toBeLessThan(hostCalls.indexOf(floodHosts[1])) + }) + + it('joins an in-flight refresh even after a stale cached host becomes fresh for a later query', async () => { + const tracker = 'https://refresh-tracker.example' + const cachedHost = 'https://cached-refresh.example' + const lateHost = 'https://late-refresh.example' + const service = 'ls_refresh' + const receipt = await slapReceipt(130, lateHost, service) + const cachedBeef = makeBeef(61) + const lateBeef = makeBeef(62) + let finishRefresh: (() => void) | undefined + const lookup = jest.fn(async (host: string) => { + if (host === tracker) { + await new Promise(resolve => { + finishRefresh = resolve + }) + return { type: 'output-list' as const, outputs: [receipt] } + } + return { + type: 'output-list' as const, + outputs: [{ beef: host === cachedHost ? cachedBeef : lateBeef, outputIndex: 0 }] + } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + ;(resolver as any).hostsCache.set(service, { hosts: [cachedHost], expiresAt: 0 }) + + const first = resolver.query$({ service, query: { caller: 1 } })[Symbol.asyncIterator]() + const firstPending = first.next() + await Promise.resolve() + ;(resolver as any).hostsCache.set(service, { + hosts: [cachedHost], + expiresAt: Date.now() + 60_000 + }) + const secondProgress: LookupAnswerProgress[] = [] + const second = (async () => { + for await (const progress of resolver.query$({ service, query: { caller: 2 } })) { + secondProgress.push(progress) + } + })() + + finishRefresh?.() + await jest.runAllTimersAsync() + await firstPending + await first.return?.() + await second + + expect(lookup.mock.calls.filter(([host]) => host === tracker)).toHaveLength(1) + expect(secondProgress.at(-1)?.outputs).toEqual( + expect.arrayContaining([ + { beef: cachedBeef, outputIndex: 0 }, + { beef: lateBeef, outputIndex: 0 } + ]) + ) + }) + + it('bounds tracker work by trackerConcurrency', async () => { + const trackers = [ + 'https://tracker-one.example', + 'https://tracker-two.example', + 'https://tracker-three.example' + ] + const service = 'ls_tracker_bound' + const host = 'https://tracker-bound-host.example' + const receipt = await slapReceipt(140, host, service) + let active = 0 + let peak = 0 + const resolver = new LookupResolver({ + facilitator: { + lookup: async (url: string) => { + if (trackers.includes(url)) { + active++ + peak = Math.max(peak, active) + await later(20) + active-- + return { type: 'output-list' as const, outputs: url === trackers[0] ? [receipt] : [] } + } + return { type: 'output-list' as const, outputs: [] } + } + }, + slapTrackers: trackers + }) + const pending = resolver.query({ service, query: {} }, undefined, { + limits: { trackerConcurrency: 1 } + }) + await jest.runAllTimersAsync() + await pending + expect(peak).toBe(1) + }) + + it('isolates a slow or throwing evidence listener so the terminal snapshot still arrives', async () => { + const host = 'https://listener.example' + let evidenceCalls = 0 + const resolver = new LookupResolver({ + facilitator: { + lookup: async () => ({ + type: 'output-list' as const, + outputs: [ + { beef: makeBeef(71), outputIndex: 0 }, + { beef: makeBeef(72), outputIndex: 1 } + ] + }) + }, + hostOverrides: { ls_listener: [host] } + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_listener', query: {} }, undefined, { + onEvidence: () => { + evidenceCalls++ + if (evidenceCalls === 1) return new Promise(() => {}) + throw new Error('consumer failed') + } + })) + progress.push(item) + })() + await jest.runAllTimersAsync() + await pending + expect(progress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + successfulHosts: 1 + }) + expect(evidenceCalls).toBe(2) + }) + + it('does not reuse a cancelled last-subscriber discovery when its old tracker completes late', async () => { + const tracker = 'https://abandoned-tracker.example' + const staleHost = 'https://abandoned-stale.example' + const freshHost = 'https://abandoned-fresh.example' + const service = 'ls_abandoned' + const staleReceipt = await slapReceipt(150, staleHost, service) + const freshReceipt = await slapReceipt(151, freshHost, service) + let resolveOld: (() => void) | undefined + let trackerCalls = 0 + const lookup = jest.fn(async (url: string) => { + if (url === tracker) { + trackerCalls++ + if (trackerCalls === 1) { + await new Promise(resolve => { + resolveOld = resolve + }) + return { type: 'output-list' as const, outputs: [staleReceipt] } + } + return { type: 'output-list' as const, outputs: [freshReceipt] } + } + return { + type: 'output-list' as const, + outputs: [{ beef: makeBeef(url === freshHost ? 81 : 80), outputIndex: 0 }] + } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const abort = new AbortController() + const abandoned = resolver + .query$({ service, query: { attempt: 1 } }, undefined, { + signal: abort.signal + }) + [Symbol.asyncIterator]() + const terminal = abandoned.next() + await Promise.resolve() + abort.abort() + await terminal + resolveOld?.() + await Promise.resolve() + + const fresh = resolver.query({ service, query: { attempt: 2 } }) + await jest.runAllTimersAsync() + await expect(fresh).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: makeBeef(81), outputIndex: 0 }] + }) + expect(trackerCalls).toBe(2) + }) + + it('keeps the ordinary 2s host and 5s tracker attempts within the 10s query deadline', async () => { + const usefulTracker = 'https://useful-4500.example' + const hangingTracker = 'https://hanging-tracker.example' + const discoveredHost = 'https://discovered-1500.example' + const service = 'ls_default_deadline' + const receipt = await slapReceipt(160, discoveredHost, service) + const lookup = jest.fn(async (url: string, _question: unknown, timeout?: number) => { + if (url === usefulTracker) { + expect(timeout).toBe(5000) + await later(4500) + return { type: 'output-list' as const, outputs: [receipt] } + } + if (url === hangingTracker) { + expect(timeout).toBe(5000) + await new Promise(() => {}) + } + expect(timeout).toBeUndefined() + await later(1500) + return { type: 'output-list' as const, outputs: [{ beef: makeBeef(91), outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [usefulTracker, hangingTracker] + }) + const pending = resolver.query({ service, query: {} }) + await jest.advanceTimersByTimeAsync(6_100) + await expect(pending).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: makeBeef(91), outputIndex: 0 }] + }) + }) + + it('does not let a tighter-limit cache hide later tracker hosts from a subsequent default query', async () => { + const tracker = 'https://tight-cache-tracker.example' + const firstHost = 'https://tight-cache-a.example' + const laterHost = 'https://tight-cache-b.example' + const service = 'ls_tight_cache' + const firstReceipt = await slapReceipt(170, firstHost, service) + const laterReceipt = await slapReceipt(171, laterHost, service) + const firstBeef = makeBeef(101) + const laterBeef = makeBeef(102) + const lookup = jest.fn(async (url: string) => { + if (url === tracker) { + return { type: 'output-list' as const, outputs: [firstReceipt, laterReceipt] } + } + return { + type: 'output-list' as const, + outputs: [{ beef: url === firstHost ? firstBeef : laterBeef, outputIndex: 0 }] + } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + + const tight = resolver.query({ service, query: { n: 1 } }, undefined, { + limits: { maxHosts: 1 } + }) + await jest.runAllTimersAsync() + await tight + + const hostCallsAfterTight = lookup.mock.calls + .map(([url]) => url) + .filter((url: string) => url === firstHost || url === laterHost) + expect(hostCallsAfterTight).toHaveLength(1) + + const full = resolver.query({ service, query: { n: 2 } }) + await jest.runAllTimersAsync() + const answer = await full + expect(answer.outputs).toEqual( + expect.arrayContaining([ + { beef: firstBeef, outputIndex: 0 }, + { beef: laterBeef, outputIndex: 0 } + ]) + ) + expect(answer.outputs).toHaveLength(2) + expect(lookup.mock.calls.filter(([url]) => url === tracker).length).toBeGreaterThan(1) + expect(lookup.mock.calls.map(([url]) => url)).toEqual( + expect.arrayContaining([firstHost, laterHost]) + ) + }) + + it('does not coalesce in-flight discovery across distinct caller limits', async () => { + const tracker = 'https://inflight-key-tracker.example' + const host = 'https://inflight-key-host.example' + const service = 'ls_inflight_key' + const receipt = await slapReceipt(180, host, service) + let finishTracker: (() => void) | undefined + const trackerGate = new Promise(resolve => { + finishTracker = resolve + }) + const lookup = jest.fn(async (url: string) => { + if (url === tracker) { + await trackerGate + return { type: 'output-list' as const, outputs: [receipt] } + } + return { type: 'output-list' as const, outputs: [] } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const first = resolver + .query$({ service, query: { n: 1 } }, undefined, { + limits: { maxHosts: 1 } + }) + [Symbol.asyncIterator]() + const firstPending = first.next() + await Promise.resolve() + const second = resolver + .query$({ service, query: { n: 2 } }, undefined, { + limits: { maxHosts: 2 } + }) + [Symbol.asyncIterator]() + const secondPending = second.next() + await Promise.resolve() + + expect((resolver as any).hostsInFlight.size).toBe(2) + const keys = Array.from((resolver as any).hostsInFlight.keys()) as string[] + expect(keys).toHaveLength(2) + expect(keys[0]).not.toEqual(keys[1]) + expect(keys.every(key => key.includes(service))).toBe(true) + expect(lookup.mock.calls.filter(([url]) => url === tracker)).toHaveLength(2) + + finishTracker?.() + await jest.runAllTimersAsync() + await firstPending + await secondPending + await first.return?.() + await second.return?.() + }) + + it('throws a resource-limit error when discovery exhausts the byte budget before any host is admitted', async () => { + const tracker = 'https://discovery-limit-tracker.example' + const lookup = jest.fn( + async ( + _url: string, + _question: unknown, + _timeout: unknown, + _signal?: AbortSignal, + options?: { consumeBytes?: (bytes: number) => void } + ) => { + options?.consumeBytes?.(4096) + return { type: 'output-list' as const, outputs: [] } + } + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [tracker] + }) + const pending = resolver.queryDetailed( + { service: 'ls_discovery_limit', query: {} }, + undefined, + { limits: { maxTotalBytes: 1024 } } + ) + pending.catch(() => { + /* asserted below */ + }) + + await jest.runAllTimersAsync() + + await expect(pending).rejects.toBeInstanceOf(LookupResourceLimitError) + await expect(pending).rejects.toMatchObject({ + name: 'LookupResourceLimitError', + limit: 'maxTotalBytes' + }) + }) + + it('throws from query() when a deadline expires before any host is admitted', async () => { + const tracker = 'https://deadline-miss-tracker.example' + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [tracker] + }) + const pending = expect( + resolver.query({ service: 'ls_deadline_miss', query: {} }, undefined, { + deadlineMs: 25 + }) + ).rejects.toThrow( + 'No competent mainnet hosts found by the SLAP trackers for lookup service: ls_deadline_miss' + ) + await jest.advanceTimersByTimeAsync(25) + await pending + }) + + it('keeps query$ deadline snapshots when no host was admitted while Promise callers still throw', async () => { + const tracker = 'https://deadline-snapshot-tracker.example' + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [tracker] + }) + const received: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$( + { service: 'ls_deadline_snapshot', query: {} }, + undefined, + { deadlineMs: 25 } + )) { + received.push(item) + } + })() + await jest.advanceTimersByTimeAsync(25) + await pending + expect(received).toHaveLength(1) + expect(received[0]).toMatchObject({ + isFinal: true, + terminalReason: 'deadline', + hostCount: 0, + outputs: [] + }) + }) + + it('reuses a covering cache that omitted optional discovery metadata', async () => { + const host = 'https://cached-meta.example' + const tracker = 'https://cached-meta-tracker.example' + const service = 'ls_cached_meta' + const beef = makeBeef(201) + const lookup = jest.fn(async (url: string) => { + if (url === tracker) throw new Error('tracker should not run') + return { type: 'output-list' as const, outputs: [{ beef, outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const limits = (resolver as any).limits + ;(resolver as any).hostsCache.set(service, { + maxHosts: limits.maxHosts, + maxHostsPerTracker: limits.maxHostsPerTracker, + maxTrackers: limits.maxTrackers, + maxResponseBytes: limits.maxResponseBytes, + maxTotalBytes: limits.maxTotalBytes, + maxOutputs: limits.maxOutputs, + hosts: [host], + expiresAt: Date.now() + 60_000 + }) + const pending = resolver.query({ service, query: {} }) + await jest.runAllTimersAsync() + await expect(pending).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef, outputIndex: 0 }] + }) + expect(lookup.mock.calls.map(([url]) => url)).toEqual([host]) + }) + + it('refreshes when a planted cache is missing any discovery bound', async () => { + const tracker = 'https://missing-bound-tracker.example' + const host = 'https://missing-bound-host.example' + const receipt = await slapReceipt(210, host, 'ls_missing_bound') + const lookup = jest.fn(async (url: string) => { + if (url === tracker) return { type: 'output-list' as const, outputs: [receipt] } + return { type: 'output-list' as const, outputs: [] } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const limits = (resolver as any).limits + const missingFields = [ + 'maxHosts', + 'maxHostsPerTracker', + 'maxTrackers', + 'maxResponseBytes', + 'maxTotalBytes', + 'maxOutputs' + ] as const + for (const missing of missingFields) { + const service = `ls_missing_${missing}` + const cached: Record = { + maxHosts: limits.maxHosts, + maxHostsPerTracker: limits.maxHostsPerTracker, + maxTrackers: limits.maxTrackers, + maxResponseBytes: limits.maxResponseBytes, + maxTotalBytes: limits.maxTotalBytes, + maxOutputs: limits.maxOutputs, + hosts: [host], + expiresAt: Date.now() + 60_000 + } + delete cached[missing] + ;(resolver as any).hostsCache.set(service, cached) + const pending = resolver.query({ service, query: {} }) + await jest.runAllTimersAsync() + await pending + } + expect(lookup.mock.calls.filter(([url]) => url === tracker)).toHaveLength(missingFields.length) + }) + + it('rejects a deadline outside the accepted range', async () => { + const resolver = new LookupResolver({ + facilitator: { lookup: async () => ({ type: 'output-list' as const, outputs: [] }) }, + hostOverrides: { ls_deadline_range: ['https://deadline-range.example'] } + }) + await expect( + resolver.query({ service: 'ls_deadline_range', query: {} }, undefined, { deadlineMs: -1 }) + ).rejects.toBeInstanceOf(RangeError) + await expect( + resolver.query({ service: 'ls_deadline_range', query: {} }, undefined, { + deadlineMs: 2_147_483_648 + }) + ).rejects.toBeInstanceOf(RangeError) + await expect( + resolver.query({ service: 'ls_deadline_range', query: {} }, undefined, { + deadlineMs: Number.NaN + }) + ).rejects.toBeInstanceOf(RangeError) + }) + + it('cancels immediately when the caller signal is already aborted', async () => { + const lookup = jest.fn(async () => ({ type: 'output-list' as const, outputs: [] })) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_preabort: ['https://preabort.example'] } + }) + const controller = new AbortController() + controller.abort() + const received: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_preabort', query: {} }, undefined, { + signal: controller.signal + })) { + received.push(item) + } + })() + await jest.runAllTimersAsync() + await pending + expect(lookup).not.toHaveBeenCalled() + expect(received.at(-1)).toMatchObject({ isFinal: true, terminalReason: 'cancelled' }) + }) + + it('aborts an in-flight query$ when the iterator throws', async () => { + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_iter_throw: ['https://iter-throw.example'] } + }) + const iterator = resolver + .query$({ service: 'ls_iter_throw', query: {} }) + [Symbol.asyncIterator]() + const first = iterator.next() + await Promise.resolve() + await expect(iterator.throw(new Error('iterator failed'))).rejects.toThrow('iterator failed') + await first.catch(() => undefined) + }) + + it('shallow-copies a question that structuredClone cannot clone for a custom facilitator', async () => { + const query: { nested: { n: number }; fn?: () => number } = { nested: { n: 1 }, fn: () => 1 } + const lookup = jest.fn(async (_url: string, question: { query: typeof query }) => { + expect(question.query).toEqual(query) + expect(question.query).not.toBe(query) + expect(question.query.nested).toBe(query.nested) + return { type: 'output-list' as const, outputs: [] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_clone: ['https://clone.example'] } + }) + const pending = resolver.query({ service: 'ls_clone', query }) + await jest.runAllTimersAsync() + await pending + expect(lookup).toHaveBeenCalledTimes(1) + }) + + it('re-serializes a non-cloneable question for the HTTPS facilitator', async () => { + const fetchClient = jest.fn( + async () => + new Response(JSON.stringify({ type: 'output-list', outputs: [] }), { + status: 200, + headers: { 'content-type': 'application/json' } + }) + ) + const resolver = new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(fetchClient as any, true), + hostOverrides: { ls_https_clone: ['https://https-clone.example'] } + }) + const pending = resolver.query({ + service: 'ls_https_clone', + query: { fn: () => 1 } + } as any) + await jest.runAllTimersAsync() + await pending + expect(fetchClient).toHaveBeenCalled() + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts new file mode 100644 index 000000000..d90721793 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts @@ -0,0 +1,272 @@ +import { createServer } from 'node:http' +import type { Server } from 'node:http' +import type { Socket } from 'node:net' +import LookupResolver, { HTTPSOverlayLookupFacilitator } from '../LookupResolver.js' +import { getOverlayHostReputationTracker } from '../HostReputationTracker.js' +import { Beef, Transaction } from '../../transaction/index.js' +import { LockingScript } from '../../script/index.js' + +interface LookupServer { + url: string + close: () => Promise +} + +/** + * This uses SDK serialization to make a structurally parseable receipt. It is + * deliberately synthetic: parsing BEEF here does not make a cryptographic or + * chain-validity claim. + */ +function structuralOutputListFixture(scriptBytes = 48 * 1024): { + answer: { + type: 'output-list' + outputs: Array<{ beef: number[]; outputIndex: number; context: number[] }> + } + evidenceBytes: number + wire: Buffer +} { + const transaction = new Transaction( + 1, + [], + [{ lockingScript: LockingScript.fromHex('00'.repeat(scriptBytes)), satoshis: 1 }], + 0 + ) + const beef = Beef.fromBinary(transaction.toBEEF()).toBinary() + const answer = { type: 'output-list' as const, outputs: [{ beef, outputIndex: 0 }] } + return { + answer, + evidenceBytes: beef.length, + wire: Buffer.from(JSON.stringify(answer)) + } +} + +function configuredResolver(url: string): LookupResolver { + return new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(fetch, true), + hostOverrides: { ls_http: [url] } + }) +} + +async function startLookupServer( + handler: Parameters[0] +): Promise { + const server = createServer(handler) + const sockets = new Set() + server.on('connection', socket => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + }) + + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + const address = server.address() + if (address === null || typeof address === 'string') + throw new Error('Expected a TCP server address') + + return { + url: `http://127.0.0.1:${address.port}`, + close: async () => { + for (const socket of sockets) socket.destroy() + await new Promise((resolve, reject) => { + ;(server as Server).close(error => (error === undefined ? resolve() : reject(error))) + }) + } + } +} + +async function waitForClose(close: Promise): Promise { + let timer: ReturnType | undefined + try { + await Promise.race([ + close, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('server did not observe client close')), 1500) + }) + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } +} + +describe('HTTPSOverlayLookupFacilitator HTTP transport', () => { + let server: LookupServer | undefined + + afterEach(async () => { + await server?.close() + server = undefined + getOverlayHostReputationTracker().reset() + }) + + it('accepts a slowly streamed structural BEEF receipt within a configured budget', async () => { + const fixture = structuralOutputListFixture() + server = await startLookupServer(async (_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + for (let offset = 0; offset < fixture.wire.length; offset += 1024) { + response.write(fixture.wire.subarray(offset, offset + 1024)) + await new Promise(resolve => setTimeout(resolve, 2)) + } + response.end() + }) + const resolver = configuredResolver(server.url) + const startedAt = Date.now() + const result = await resolver.queryDetailed({ service: 'ls_http', query: {} }, 2000, { + limits: { maxResponseBytes: fixture.wire.length, maxTotalBytes: fixture.wire.length } + }) + const elapsedMs = Date.now() - startedAt + + expect(result.answer.outputs).toHaveLength(1) + expect(result.answer.outputs[0].beef).toEqual(fixture.answer.outputs[0].beef) + expect(result.progress.receivedBytes).toBe(fixture.wire.length) + expect(fixture.evidenceBytes).toBeGreaterThanOrEqual(48 * 1024) + expect(fixture.wire.length).toBeGreaterThan(fixture.evidenceBytes) + expect({ evidenceBytes: fixture.evidenceBytes, jsonWireBytes: fixture.wire.length }).toEqual({ + evidenceBytes: 49_180, + jsonWireBytes: 98_429 + }) + expect(elapsedMs).toBeGreaterThanOrEqual(20) + expect(elapsedMs).toBeLessThan(2000) + }) + + it('limits evidence callbacks for the structural receipt, then admits it through evidenceLimits', async () => { + const fixture = structuralOutputListFixture() + server = await startLookupServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(fixture.wire) + }) + const limitedEvidence: string[] = [] + const limited = await configuredResolver(server.url).queryDetailed( + { service: 'ls_http', query: {} }, + 1000, + { + limits: { + maxResponseBytes: fixture.wire.length, + maxTotalBytes: fixture.wire.length, + maxEvidenceBytes: fixture.evidenceBytes - 1 + }, + onEvidence: event => limitedEvidence.push(event.type) + } + ) + const admittedEvidence: string[] = [] + const admitted = await configuredResolver(server.url).queryDetailed( + { service: 'ls_http', query: {} }, + 1000, + { + limits: { maxResponseBytes: fixture.wire.length, maxTotalBytes: fixture.wire.length }, + evidenceLimits: { maxBytes: fixture.evidenceBytes }, + onEvidence: event => admittedEvidence.push(event.type) + } + ) + + expect(limitedEvidence).toEqual(['limit']) + expect(limited.progress.terminalReason).toBe('resource-limit') + expect(limited.progress.limitsHit).toContain('maxEvidenceBytes') + expect(admittedEvidence).toEqual(['output']) + expect(admitted.progress.terminalReason).toBe('settled') + expect(admitted.progress.receivedBytes).toBe(fixture.wire.length) + }) + + it.each(['maxResponseBytes', 'maxTotalBytes'] as const)( + 'reports %s exhaustion without recording an availability failure', + async limitName => { + const fixture = structuralOutputListFixture() + server = await startLookupServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(fixture.wire) + }) + const smallLimit = fixture.wire.length - 1 + const result = await configuredResolver(server.url).queryDetailed( + { service: 'ls_http', query: {} }, + 1000, + { + limits: + limitName === 'maxResponseBytes' + ? { maxResponseBytes: smallLimit, maxTotalBytes: fixture.wire.length } + : { maxResponseBytes: fixture.wire.length, maxTotalBytes: smallLimit } + } + ) + + expect(result.answer.outputs).toEqual([]) + expect(result.progress.terminalReason).toBe('resource-limit') + expect(result.progress.limitsHit).toContain(limitName) + expect(result.progress.completedHosts).toBe(1) + expect(getOverlayHostReputationTracker().snapshot(server.url)).toMatchObject({ + totalFailures: 0, + consecutiveFailures: 0, + totalSuccesses: 0 + }) + } + ) + + it('times out a response whose body never finishes and closes the server-side response', async () => { + let resolveClosed: () => void = () => undefined + const responseClosed = new Promise(resolve => { + resolveClosed = resolve + }) + server = await startLookupServer((_request, response) => { + response.once('close', resolveClosed) + response.writeHead(200, { 'content-type': 'application/json' }) + response.write('{"type":"freeform","result":"') + }) + const facilitator = new HTTPSOverlayLookupFacilitator(fetch, true) + + await expect( + facilitator.lookup(server.url, { service: 'ls_http', query: {} }, 50, undefined, { + maxResponseBytes: 1024, + maxOutputs: 1 + }) + ).rejects.toThrow('Request timed out') + + await waitForClose(responseClosed) + }) + + it.each([ + ['invalid UTF-8', Buffer.from([0xff, 0xfe])], + ['malformed JSON', Buffer.from('{')] + ])('rejects %s without treating it as a successful answer', async (_name, payload) => { + server = await startLookupServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(payload) + }) + const facilitator = new HTTPSOverlayLookupFacilitator(fetch, true) + + await expect( + facilitator.lookup(server.url, { service: 'ls_http', query: {} }, 1000, undefined, { + maxResponseBytes: 1024, + maxOutputs: 1 + }) + ).rejects.toBeInstanceOf(Error) + }) + + it('honors an early caller abort and closes the server-side response', async () => { + let resolveStarted: () => void = () => undefined + const started = new Promise(resolve => { + resolveStarted = resolve + }) + let resolveClosed: () => void = () => undefined + const responseClosed = new Promise(resolve => { + resolveClosed = resolve + }) + server = await startLookupServer((_request, response) => { + response.once('close', resolveClosed) + response.writeHead(200, { 'content-type': 'application/json' }) + response.write('{"type":"freeform","result":"') + resolveStarted() + }) + const facilitator = new HTTPSOverlayLookupFacilitator(fetch, true) + const controller = new AbortController() + const lookup = facilitator.lookup( + server.url, + { service: 'ls_http', query: {} }, + 2000, + controller.signal, + { maxResponseBytes: 1024, maxOutputs: 1 } + ) + + await started + controller.abort() + + await expect(lookup).rejects.toMatchObject({ name: 'AbortError', message: 'Lookup cancelled' }) + await waitForClose(responseClosed) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts new file mode 100644 index 000000000..6ebb95d1c --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts @@ -0,0 +1,567 @@ +import LookupResolver, { + DEFAULT_LOOKUP_LIMITS, + LookupResourceLimitError, + type LookupAnswer, + type LookupAnswerProgress, + type LookupEvidenceEvent, + type LookupFacilitatorAnswer, + type LookupQuestion, + type LookupRequestOptions, + type UnreachableHostInfo +} from '../LookupResolver' +import { getOverlayHostReputationTracker } from '../HostReputationTracker' +import OverlayAdminTokenTemplate from '../OverlayAdminTokenTemplate' +import { CompletedProtoWallet } from '../../auth/certificates/__tests/CompletedProtoWallet' +import { PrivateKey } from '../../primitives/index' +import { LockingScript } from '../../script/index' +import { Transaction } from '../../transaction/index' + +const service = 'ls_limits' +const question: LookupQuestion = { service, query: { id: 1 } } + +type LookupOutput = LookupAnswer['outputs'][number] + +/** Structurally parseable receipt; no chain-validity claim is made here. */ +function receipt(satoshis: number): LookupOutput { + const tx = new Transaction(1, [], [{ lockingScript: LockingScript.fromHex('88'), satoshis }], 0) + return { beef: tx.toBEEF(), outputIndex: 0 } +} + +/** A SLAP advertisement naming `domain` as a host for `advertised`. */ +async function slapAdvertisement( + scalar: number, + domain: string, + advertised: string +): Promise { + const wallet = new CompletedProtoWallet(new PrivateKey(scalar)) + const template = new OverlayAdminTokenTemplate(wallet) + const lockingScript = await template.lock('SLAP', domain, advertised) + const tx = new Transaction(1, [], [{ lockingScript, satoshis: 1 }], 0) + return { beef: tx.toBEEF(), outputIndex: 0 } +} + +interface Deferred { + promise: Promise + resolve: (value: T) => void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + const promise = new Promise(resolvePromise => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +async function collect( + progress: AsyncIterable +): Promise { + const emissions: LookupAnswerProgress[] = [] + for await (const emission of progress) emissions.push(emission) + return emissions +} + +async function finalEmission( + progress: AsyncIterable +): Promise { + const emissions = await collect(progress) + const last = emissions.at(-1) + if (last === undefined) throw new Error('expected at least one emission') + return last +} + +async function caught(work: Promise): Promise { + return await work.then( + () => { + throw new Error('expected the query to reject') + }, + (error: unknown) => error + ) +} + +/** One macrotask turn: every pending microtask continuation has run. */ +async function eventLoopTurn(): Promise { + await new Promise(resolve => setTimeout(resolve, 0)) +} + +describe('LookupResolver query resource bounds', () => { + beforeEach(() => { + getOverlayHostReputationTracker().reset() + }) + + afterEach(() => { + getOverlayHostReputationTracker().reset() + }) + + it('stops merging at maxOutputs once a second host contributes a new outpoint', async () => { + const answers: Record = { + 'https://first.example': { type: 'output-list', outputs: [receipt(1)] }, + 'https://second.example': { type: 'output-list', outputs: [receipt(2)] } + } + const resolver = new LookupResolver({ + hostOverrides: { [service]: Object.keys(answers) }, + limits: { maxOutputs: 1 }, + facilitator: { lookup: async host => answers[host] } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(answer.outputs).toHaveLength(1) + expect(progress.successfulHosts).toBe(2) + expect(progress.limitsHit).toContain('maxOutputs') + expect(progress.terminalReason).toBe('resource-limit') + }) + + it('defaults the evidence byte budget when only an output count is supplied', async () => { + const events: LookupEvidenceEvent[] = [] + const first = receipt(1) + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://evidence.example'] }, + facilitator: { + lookup: async () => ({ type: 'output-list', outputs: [first, receipt(2)] }) + } + }) + + const { progress } = await resolver.queryDetailed(question, undefined, { + evidenceLimits: { maxOutputs: 1 }, + onEvidence: event => { + events.push(event) + } + }) + + expect(events.filter(event => event.type === 'output')).toHaveLength(1) + expect(events.at(-1)).toEqual({ type: 'limit' }) + expect(progress.limitsHit).toContain('maxEvidenceOutputs') + // The default byte budget is generous: the single receipt was admitted whole. + expect(progress.evidenceBytes).toBe(first.beef.length) + // Evidence intake is additive; legacy aggregation still merged both outputs. + expect(progress.outputs).toHaveLength(2) + }) + + it('bounds the candidate scan and skips malformed and over-budget host entries', async () => { + const queried: string[] = [] + const resolver = new LookupResolver({ + hostOverrides: { + [service]: [ + 'not a url', + 'https://h1.example', + 'https://h2.example', + 'https://h3.example', + 'https://h4.example', + 'https://h5.example' + ] + }, + limits: { maxHosts: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [receipt(1)] } + } + } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(queried).toEqual(['https://h1.example']) + expect(answer.outputs).toHaveLength(1) + expect(progress.discoveredHosts).toBe(1) + // 2 beyond the maxHosts * 4 scan window, 1 unparseable, 2 past maxHosts. + expect(progress.skippedHosts).toBe(5) + expect(progress.limitsHit).toContain('maxHosts') + }) + + it('cancelling from the first limit notification leaves every host unqueried', async () => { + const controller = new AbortController() + const queried: string[] = [] + const events: LookupEvidenceEvent[] = [] + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://one.example', 'https://two.example'] }, + additionalHosts: { [service]: ['https://three.example'] }, + limits: { maxHosts: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [] } + } + } + }) + + const error = await caught( + resolver.query(question, undefined, { + signal: controller.signal, + onEvidence: event => { + events.push(event) + if (event.type === 'limit') controller.abort() + } + }) + ) + + expect((error as Error).name).toBe('AbortError') + expect(queried).toEqual([]) + // Exactly one limit notification, even though cancellation records its own. + expect(events).toEqual([{ type: 'limit' }]) + }) + + it('reserves a per-source quota for additional hosts while discovery is refreshed', async () => { + const queried: string[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: ['https://tracker.example'], + additionalHosts: { [service]: ['https://add1.example', 'https://add2.example'] }, + limits: { maxHosts: 2, maxTrackers: 1 }, + facilitator: { + lookup: async (host, asked) => { + queried.push(host) + if (asked.service === 'ls_slap') return { type: 'output-list', outputs: [] } + return { type: 'output-list', outputs: [receipt(1)] } + } + } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(queried).toEqual(['https://add1.example', 'https://tracker.example']) + expect(answer.outputs).toHaveLength(1) + expect(progress.discoveredHosts).toBe(1) + expect(progress.skippedHosts).toBe(1) + expect(progress.limitsHit).toContain('maxHosts') + }) + + it('cancelling on the quota limit releases discovery before any tracker or host is contacted', async () => { + const controller = new AbortController() + const queried: string[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: ['https://tracker.example'], + additionalHosts: { [service]: ['https://add1.example', 'https://add2.example'] }, + limits: { maxHosts: 2, maxTrackers: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [] } + } + } + }) + + const error = await caught( + resolver.query(question, undefined, { + signal: controller.signal, + onEvidence: event => { + if (event.type === 'limit') controller.abort() + } + }) + ) + await eventLoopTurn() + + expect((error as Error).name).toBe('AbortError') + // The additional host was already dispatched and the SLAP refresh was about + // to subscribe; cancellation must reach both before either sends a request. + expect(queried).toEqual([]) + }) + + it('fails closed when SLAP discovery bytes would exceed the aggregate budget', async () => { + const hostCharged = deferred() + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: ['https://tracker.example'], + additionalHosts: { [service]: ['https://add.example'] }, + limits: { maxHosts: 4, maxTrackers: 1, maxTotalBytes: 100 }, + facilitator: { + lookup: async ( + _host: string, + asked: LookupQuestion, + _timeout?: number, + _signal?: AbortSignal, + options?: LookupRequestOptions + ) => { + if (asked.service === 'ls_slap') { + await hostCharged.promise + await eventLoopTurn() + options?.consumeBytes?.(30) + return { type: 'output-list', outputs: [] } + } + options?.consumeBytes?.(80) + hostCharged.resolve() + return { type: 'output-list', outputs: [] } + } + } + }) + + const { progress } = await resolver.queryDetailed(question) + + expect(progress.hostCount).toBe(1) + expect(progress.successfulHosts).toBe(1) + // 80 host bytes were accepted; the 30 discovery bytes that would have + // breached maxTotalBytes are refused and never credited. + expect(progress.receivedBytes).toBe(80) + expect(progress.limitsHit).toContain('maxTotalBytes') + expect(progress.terminalReason).toBe('resource-limit') + expect(progress.discoveryComplete).toBe(false) + }) + + it('refuses byte reports that arrive after the query was cancelled', async () => { + const controller = new AbortController() + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://late.example'] }, + limits: { maxTotalBytes: 1024 }, + facilitator: { + lookup: async ( + _host: string, + _asked: LookupQuestion, + _timeout?: number, + _signal?: AbortSignal, + options?: LookupRequestOptions + ) => { + options?.consumeBytes?.(4) + controller.abort() + options?.consumeBytes?.(4) + return { type: 'output-list', outputs: [] } + } + } + }) + + const final = await finalEmission( + resolver.query$(question, undefined, { signal: controller.signal }) + ) + + expect(final.terminalReason).toBe('cancelled') + expect(final.receivedBytes).toBe(4) + }) + + it('cancelling mid-flight drops the in-flight peer answer and skips queued hosts', async () => { + const controller = new AbortController() + const queried: string[] = [] + const hosts = ['https://q1.example', 'https://q2.example', 'https://q3.example'] + const resolver = new LookupResolver({ + hostOverrides: { [service]: hosts }, + limits: { maxHosts: 3, hostConcurrency: 2 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [receipt(hosts.indexOf(host) + 1)] } + } + } + }) + + const final = await finalEmission( + resolver.query$(question, undefined, { + signal: controller.signal, + onEvidence: event => { + if (event.type === 'output') controller.abort() + } + }) + ) + + expect(queried).toEqual(['https://q1.example', 'https://q2.example']) + expect(final.terminalReason).toBe('cancelled') + expect(final.hostCount).toBe(2) + // The third host never left the queue, and neither in-flight answer was + // aggregated: a cancelled attempt never answered the question. + expect(final.skippedHosts).toBe(1) + expect(final.successfulHosts).toBe(0) + expect(final.outputs).toEqual([]) + }) + + it('treats a second cancellation of the same query as a no-op', async () => { + const controller = new AbortController() + const events: LookupEvidenceEvent[] = [] + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://stalled.example'] }, + facilitator: { + lookup: async () => await new Promise(() => {}) + } + }) + + const iterator = resolver + .query$(question, undefined, { + signal: controller.signal, + softTimeoutMs: 0, + graceMs: 0, + onEvidence: event => { + events.push(event) + } + }) + [Symbol.asyncIterator]() + + const first = await iterator.next() + expect(first.done).toBe(false) + expect(first.value.isFinal).toBe(false) + expect(first.value.hostCount).toBe(1) + + controller.abort() + // Breaking the iterator cancels a second time through the iterator signal. + await iterator.return?.(undefined) + + expect(events).toEqual([{ type: 'limit' }]) + }) + + it('refuses a new query once the concurrent query ceiling is reached', async () => { + const gate = deferred() + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://capped.example'] }, + facilitator: { lookup: async () => await gate.promise } + }) + + const iterators = Array.from({ length: 128 }, () => + resolver.query$(question)[Symbol.asyncIterator]() + ) + const pending = iterators.map(async iterator => await iterator.next()) + await eventLoopTurn() + + const error = await caught(resolver.query(question)) + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('activeQueries') + + gate.resolve({ type: 'output-list', outputs: [] }) + await Promise.all(pending) + await Promise.all( + iterators.map(async iterator => { + await iterator.return?.(undefined) + }) + ) + + // Every finished query released its slot. + const released = await resolver.queryDetailed(question) + expect(released.progress.hostCount).toBe(1) + }) + + it('drops an answer whose receipts exceed maxResponseBytes without blaming the host', async () => { + const host = 'https://oversized.example' + const resolver = new LookupResolver({ + hostOverrides: { [service]: [host] }, + limits: { maxResponseBytes: 16 }, + facilitator: { lookup: async () => ({ type: 'output-list', outputs: [receipt(1)] }) } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(answer.outputs).toEqual([]) + expect(progress.limitsHit).toEqual(['maxResponseBytes']) + expect(progress.terminalReason).toBe('resource-limit') + // A client-side budget rejection is not an availability failure. + expect(progress.failedHosts).toBe(0) + expect(progress.rejectedHosts).toBe(0) + expect(getOverlayHostReputationTracker().snapshot(host)?.totalFailures).toBe(0) + }) + + it('drops an answer with more outputs than maxOutputs instead of truncating it', async () => { + const host = 'https://overcounted.example' + const resolver = new LookupResolver({ + hostOverrides: { [service]: [host] }, + limits: { maxOutputs: 1 }, + facilitator: { + lookup: async () => ({ type: 'output-list', outputs: [receipt(1), receipt(2)] }) + } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(answer.outputs).toEqual([]) + expect(progress.limitsHit).toEqual(['maxOutputs']) + expect(progress.failedHosts).toBe(0) + expect(getOverlayHostReputationTracker().snapshot(host)?.totalFailures).toBe(0) + }) + + it('queries only the budgeted number of SLAP trackers and names the limit it hit', async () => { + const trackers = ['https://t1.example', 'https://t2.example'] + const queried: string[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: trackers, + limits: { maxTrackers: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [] } + } + } + }) + + const error = await caught(resolver.query(question)) + + expect(queried).toEqual(['https://t1.example']) + // A budget exhausted during discovery keeps its own error rather than + // borrowing the no-competent-hosts message. + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('maxTrackers') + }) + + it('evicts the oldest SLAP attribution once the advertisement map is full', async () => { + const tracker = 'https://ad-tracker.example' + const advertisements = await Promise.all([ + slapAdvertisement(11, 'https://adv1.example', service), + slapAdvertisement(12, 'https://adv2.example', service), + slapAdvertisement(13, 'https://adv3.example', service) + ]) + const unreachable: UnreachableHostInfo[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: [tracker], + cache: { hostsMaxEntries: 1 }, + limits: { maxHosts: 2, maxTrackers: 1 }, + facilitator: { + lookup: async (_host, asked) => { + if (asked.service === 'ls_slap') { + return { type: 'output-list', outputs: advertisements } + } + throw new Error('connection refused') + } + } + }) + + const { progress } = await resolver.queryDetailed(question, undefined, { + onUnreachableHost: info => { + unreachable.push(info) + } + }) + + expect(progress.failedHosts).toBe(2) + const attribution = new Map(unreachable.map(info => [info.host, info.advertisedBy])) + expect(attribution.size).toBe(2) + expect(attribution.get('https://adv2.example')).toBe(tracker) + // adv1 was evicted when adv3's attribution arrived, so it reports no tracker. + expect(attribution.has('https://adv1.example')).toBe(true) + expect(attribution.get('https://adv1.example')).toBeUndefined() + }) + + it('keeps a still-fresh broader host cache when a tighter query rediscovers', async () => { + const tracker = 'https://cache-tracker.example' + const broad = await slapAdvertisement(21, 'https://cached.example', service) + const tight = await slapAdvertisement(22, 'https://rediscovered.example', service) + let trackerCalls = 0 + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: [tracker], + limits: { maxTrackers: 1 }, + facilitator: { + lookup: async (_host, asked) => { + if (asked.service === 'ls_slap') { + trackerCalls++ + return { type: 'output-list', outputs: [trackerCalls === 1 ? broad : tight] } + } + // An immediate-backoff failure, so the cached host stops being + // available and the tighter query must refresh discovery. + throw new Error('Failed to fetch') + } + } + }) + + const first = await resolver.queryDetailed(question) + expect(first.progress.failedHosts).toBe(1) + + const second = await resolver.queryDetailed(question, undefined, { + limits: { maxOutputs: 8 } + }) + expect(trackerCalls).toBe(2) + expect(second.progress.failedHosts).toBe(1) + + const cache = ( + resolver as unknown as { + hostsCache: Map + } + ).hostsCache + const entry = cache.get(service) + expect(entry?.hosts).toEqual(['https://cached.example']) + expect(entry?.maxOutputs).toBe(DEFAULT_LOOKUP_LIMITS.maxOutputs) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts index b0072979e..88cd1f916 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts @@ -10,6 +10,23 @@ const mockFacilitator = { lookup: jest.fn() } +const expectLookupCalls = (actual: unknown[][], expected: unknown[][]): void => { + expect(actual.map(call => call.slice(0, 3))).toEqual(expected) + for (const call of actual) { + expect(call[3]).toEqual(expect.any(AbortSignal)) + expect(call[4]).toEqual( + expect.objectContaining({ + maxResponseBytes: 32 * 1024 * 1024, + consumeBytes: expect.any(Function) + }) + ) + } +} + +const expectLookupCall = (actual: unknown[], expected: unknown[]): void => { + expectLookupCalls([actual], [expected]) +} + const sampleBeef1 = new Transaction( 1, [], @@ -96,7 +113,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -137,40 +154,24 @@ describe('LookupResolver', () => { 0 ) - mockFacilitator.lookup - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - outputIndex: 0, - beef: slapTx.toBEEF() - } - ] - }) - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - beef: sampleBeef1, - outputIndex: 0 - } - ] - }) - .mockReturnValueOnce({ + mockFacilitator.lookup.mockImplementation((url: string, question: { service: string }) => { + if (question.service === 'ls_slap') { + return { + type: 'output-list', + outputs: [{ outputIndex: 0, beef: slapTx.toBEEF() }] + } + } + if (url === 'https://slaphost.com') { + return { type: 'output-list', outputs: [{ beef: sampleBeef1, outputIndex: 0 }] } + } + return { type: 'output-list', outputs: [ - { - // duplicate the output the other host knows about - beef: sampleBeef1, - outputIndex: 0 - }, - { - // the additional host also knows about a second output - beef: sampleBeef2, - outputIndex: 1033 - } + { beef: sampleBeef1, outputIndex: 0 }, + { beef: sampleBeef2, outputIndex: 1033 } ] - }) + } + }) const r = new LookupResolver({ facilitator: mockFacilitator, @@ -198,30 +199,30 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ - 'https://mock.slap', + // additional host should also have been queried first + 'https://additional.host', { - service: 'ls_slap', + service: 'ls_foo', query: { - service: 'ls_foo' + test: 1 } }, - 5000 + undefined ], [ - 'https://slaphost.com', + 'https://mock.slap', { - service: 'ls_foo', + service: 'ls_slap', query: { - test: 1 + service: 'ls_foo' } }, - undefined + 5000 ], [ - // additional host should also have been queried - 'https://additional.host', + 'https://slaphost.com', { service: 'ls_foo', query: { @@ -264,7 +265,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://override.host', { @@ -332,7 +333,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://override.host', { @@ -357,7 +358,7 @@ describe('LookupResolver', () => { ]) }) - it('should handle multiple SLAP trackers and resolve with first responder hosts', async () => { + it('queries every eligible host advertised during the active attempt, including a later tracker', async () => { const slapHostKey1 = new PrivateKey(42) const slapWallet1 = new CompletedProtoWallet(slapHostKey1) const slapLib1 = new OverlayAdminTokenTemplate(slapWallet1) @@ -390,36 +391,28 @@ describe('LookupResolver', () => { 0 ) - // SLAP trackers return hosts — first responder wins - mockFacilitator.lookup - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - outputIndex: 0, - beef: slapTx1.toBEEF() + mockFacilitator.lookup.mockImplementation((url: string, question: { service: string }) => { + if (question.service === 'ls_slap') { + if (url === 'https://mock.slap1') { + return { + type: 'output-list', + outputs: [{ outputIndex: 0, beef: slapTx1.toBEEF() }] } - ] - }) - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - outputIndex: 0, - beef: slapTx2.toBEEF() + } + if (url === 'https://mock.slap2') { + return { + type: 'output-list', + outputs: [{ outputIndex: 0, beef: slapTx2.toBEEF() }] } - ] - }) - - // Only the first-resolved tracker's host gets queried - mockFacilitator.lookup.mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - beef: sampleBeef3, - outputIndex: 0 } - ] + } + if (url === 'https://slaphost1.com') { + return { type: 'output-list', outputs: [{ beef: sampleBeef3, outputIndex: 0 }] } + } + if (url === 'https://slaphost2.com') { + return { type: 'output-list', outputs: [{ beef: sampleBeef2, outputIndex: 1 }] } + } + throw new Error(`unexpected host ${url}`) }) const r = new LookupResolver({ @@ -432,15 +425,24 @@ describe('LookupResolver', () => { query: { test: 1 } }) - // Only the first tracker's host results are returned - expect(res).toEqual({ - type: 'output-list', - outputs: [{ beef: sampleBeef3, outputIndex: 0 }] - }) + expect(res.outputs).toEqual( + expect.arrayContaining([ + { beef: sampleBeef3, outputIndex: 0 }, + { beef: sampleBeef2, outputIndex: 1 } + ]) + ) + expect(res.outputs).toHaveLength(2) - // Both SLAP trackers are queried, but only the first host is used for the actual query - expect(mockFacilitator.lookup.mock.calls.length).toBeGreaterThanOrEqual(3) - expect(mockFacilitator.lookup.mock.calls[0]).toEqual([ + const calledUrls = mockFacilitator.lookup.mock.calls.map((call: unknown[]) => call[0]) + expect(calledUrls).toEqual( + expect.arrayContaining([ + 'https://mock.slap1', + 'https://mock.slap2', + 'https://slaphost1.com', + 'https://slaphost2.com' + ]) + ) + expectLookupCall(mockFacilitator.lookup.mock.calls[0], [ 'https://mock.slap1', { service: 'ls_slap', @@ -450,7 +452,7 @@ describe('LookupResolver', () => { }, 5000 ]) - expect(mockFacilitator.lookup.mock.calls[1]).toEqual([ + expectLookupCall(mockFacilitator.lookup.mock.calls[1], [ 'https://mock.slap2', { service: 'ls_slap', @@ -460,16 +462,6 @@ describe('LookupResolver', () => { }, 5000 ]) - expect(mockFacilitator.lookup.mock.calls[2]).toEqual([ - 'https://slaphost1.com', - { - service: 'ls_foo', - query: { - test: 1 - } - }, - undefined - ]) }) it('should de-duplicate outputs from multiple hosts', async () => { @@ -541,7 +533,7 @@ describe('LookupResolver', () => { outputs: [duplicateOutput] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -644,7 +636,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -699,7 +691,7 @@ describe('LookupResolver', () => { 'No competent mainnet hosts found by the SLAP trackers for lookup service: ls_foo' ) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -786,7 +778,7 @@ describe('LookupResolver', () => { ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -848,7 +840,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -884,7 +876,7 @@ describe('LookupResolver', () => { 'No competent mainnet hosts found by the SLAP trackers for lookup service: ls_foo' ) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1115,7 +1107,7 @@ describe('LookupResolver', () => { ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap1', { @@ -1238,7 +1230,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1341,7 +1333,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1443,7 +1435,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1522,7 +1514,7 @@ describe('LookupResolver', () => { outputs: [] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts new file mode 100644 index 000000000..d31c59394 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts @@ -0,0 +1,216 @@ +import { + HTTPSOverlayLookupFacilitator, + LookupResourceLimitError, + type LookupAnswer, + type LookupFacilitatorAnswer +} from '../LookupResolver' +import { Transaction } from '../../transaction/index' +import { LockingScript } from '../../script/index' + +const question = { service: 'ls_transport', query: { id: 1 } } +const host = 'https://transport.example' + +const jsonResponse = (body: unknown): Response => + new Response(JSON.stringify(body), { headers: { 'content-type': 'application/json' } }) + +const octetResponse = (payload: Uint8Array): Response => + new Response(payload, { headers: { 'content-type': 'application/octet-stream' } }) + +/** + * Structurally parseable receipt. Parsing BEEF here makes no chain-validity + * claim; these tests only exercise the transport's own byte and count bounds. + */ +function transaction(scriptBytes: number, satoshis = 1): Transaction { + return new Transaction( + 1, + [], + [{ lockingScript: LockingScript.fromHex('00'.repeat(scriptBytes)), satoshis }], + 0 + ) +} + +/** + * Aggregated octet-stream wire format: varint outpoint count, then per outpoint + * a 32-byte txid, a varint output index and a varint-prefixed context, followed + * by the shared BEEF. `repeats` outpoints all reference the same transaction. + */ +function octetPayload(tx: Transaction, repeats: number): Buffer { + const txid = Buffer.from(tx.id('hex'), 'hex') + const outpoints = Array.from({ length: repeats }, (_unused, index) => + Buffer.concat([txid, Buffer.from([index]), Buffer.from([0x00])]) + ) + return Buffer.concat([Buffer.from([repeats]), ...outpoints, Buffer.from(tx.toBEEF())]) +} + +function outputsOf(answer: LookupFacilitatorAnswer): LookupAnswer['outputs'] { + if (answer.type !== 'output-list') throw new Error('expected an output-list answer') + return answer.outputs +} + +async function caught(work: Promise): Promise { + return await work.then( + () => { + throw new Error('expected the lookup to reject') + }, + (error: unknown) => error + ) +} + +describe('HTTPSOverlayLookupFacilitator bounded transport', () => { + it('refuses to issue a request when the caller signal is already aborted', async () => { + const fetchClient = jest.fn() + const facilitator = new HTTPSOverlayLookupFacilitator( + fetchClient as unknown as typeof fetch, + true + ) + const controller = new AbortController() + controller.abort() + + const error = await caught(facilitator.lookup(host, question, 2000, controller.signal)) + + expect(error).toBeInstanceOf(Error) + expect((error as Error).name).toBe('AbortError') + expect((error as Error).message).toBe('Lookup cancelled') + expect(fetchClient).not.toHaveBeenCalled() + }) + + it('reports cancellation, not an HTTP failure, when the caller aborts as the response arrives', async () => { + const controller = new AbortController() + let issued: Response | undefined + const fetchClient = jest.fn(async () => { + controller.abort() + issued = new Response('service unavailable', { status: 503, statusText: 'Unavailable' }) + return issued + }) + const facilitator = new HTTPSOverlayLookupFacilitator( + fetchClient as unknown as typeof fetch, + true + ) + + const error = await caught(facilitator.lookup(host, question, 2000, controller.signal)) + // Let the in-flight request settle so its body cleanup is observable. + await new Promise(resolve => setTimeout(resolve, 0)) + + expect((error as Error).name).toBe('AbortError') + expect((error as Error).message).toBe('Lookup cancelled') + // A cancelled request must not leave the response body undrained. + expect(issued?.bodyUsed).toBe(true) + }) + + it('rejects a JSON output list longer than the requested output budget', async () => { + const answer = { + type: 'output-list', + outputs: [ + { beef: transaction(1, 1).toBEEF(), outputIndex: 0 }, + { beef: transaction(1, 2).toBEEF(), outputIndex: 0 } + ] + } + const facilitator = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => jsonResponse(answer)) as unknown as typeof fetch, + true + ) + + const error = await caught( + facilitator.lookup(host, question, 2000, undefined, { maxOutputs: 1 }) + ) + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('maxOutputs') + + // The bound is inclusive: exactly maxOutputs is still accepted. + const accepted = await facilitator.lookup(host, question, 2000, undefined, { maxOutputs: 2 }) + expect(outputsOf(accepted)).toHaveLength(2) + }) + + it('rejects an octet-stream outpoint count that is negative or over the output budget', async () => { + const tx = transaction(4) + const overBudget = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(octetPayload(tx, 3))) as unknown as typeof fetch, + true + ) + const tooMany = await caught( + overBudget.lookup(host, question, 2000, undefined, { maxOutputs: 2 }) + ) + expect(tooMany).toBeInstanceOf(LookupResourceLimitError) + expect((tooMany as LookupResourceLimitError).limit).toBe('maxOutputs') + + // 0xff + eight 0xff bytes decodes as -1: a count that must never be trusted. + const negativeCount = Buffer.concat([ + Buffer.from([0xff]), + Buffer.alloc(8, 0xff), + Buffer.from(tx.toBEEF()) + ]) + const negative = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(negativeCount)) as unknown as typeof fetch, + true + ) + const malformed = await caught( + negative.lookup(host, question, 2000, undefined, { maxOutputs: 64 }) + ) + expect(malformed).toBeInstanceOf(LookupResourceLimitError) + expect((malformed as LookupResourceLimitError).limit).toBe('maxOutputs') + }) + + it('stops octet-stream extraction once the extracted bytes exceed the response budget', async () => { + const tx = transaction(400) + const payload = octetPayload(tx, 3) + const beefBytes = tx.toBEEF().length + // Three outpoints on one transaction extract three atomic BEEF copies, so + // the retained total outruns the wire length the reader already accepted. + expect(2 * beefBytes).toBeGreaterThan(payload.length + 1) + const facilitator = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(payload)) as unknown as typeof fetch, + true + ) + + const error = await caught( + facilitator.lookup(host, question, 2000, undefined, { + maxResponseBytes: payload.length + 1, + maxOutputs: 8 + }) + ) + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('maxResponseBytes') + + const accepted = await facilitator.lookup(host, question, 2000, undefined, { + maxResponseBytes: 4 * beefBytes, + maxOutputs: 8 + }) + const outputs = outputsOf(accepted) + expect(outputs).toHaveLength(3) + expect(outputs.map(output => output.txid)).toEqual([tx.id('hex'), tx.id('hex'), tx.id('hex')]) + }) + + it('abandons octet-stream extraction when the caller cancels mid-decode', async () => { + const controller = new AbortController() + const tx = transaction(8) + const payload = octetPayload(tx, 60) + let scheduled = false + const facilitator = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(payload)) as unknown as typeof fetch, + true + ) + + const startedAt = Date.now() + const error = await caught( + facilitator.lookup(host, question, 5000, controller.signal, { + maxResponseBytes: 1_000_000, + maxOutputs: 128, + // Cancel once the transport has begun reporting bytes: extraction + // yields to the event loop between outputs and must observe the abort. + consumeBytes: () => { + if (scheduled) return + scheduled = true + setTimeout(() => controller.abort(), 0) + } + }) + ) + // The abandoned decode is detached from the caller-facing promise; give it + // an event-loop turn so its own cancellation check runs before teardown. + await new Promise(resolve => setTimeout(resolve, 30)) + + expect((error as Error).name).toBe('AbortError') + expect((error as Error).message).toBe('Lookup cancelled') + // Cancellation settles the request rather than waiting out the 5s deadline. + expect(Date.now() - startedAt).toBeLessThan(2000) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts new file mode 100644 index 000000000..4611a83d3 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts @@ -0,0 +1,64 @@ +import { + DEFAULT_LOOKUP_LIMITS, + lookupAbortError, + lookupLimits, + normalizeLookupHost, + withLookupAbort +} from '../LookupResources.js' + +describe('lookupLimits', () => { + it('accepts positive safe integers and rejects every other shape', () => { + expect(lookupLimits(undefined).maxHosts).toBe(DEFAULT_LOOKUP_LIMITS.maxHosts) + expect(lookupLimits({ maxHosts: 3 }).maxHosts).toBe(3) + expect(() => lookupLimits({ maxHosts: 0 })).toThrow(RangeError) + expect(() => lookupLimits({ maxHosts: -1 })).toThrow( + /Lookup limit maxHosts must be a positive safe integer/ + ) + expect(() => lookupLimits({ hostConcurrency: 1.5 })).toThrow(RangeError) + expect(() => lookupLimits({ maxTrackers: Number.NaN })).toThrow(RangeError) + expect(() => lookupLimits({ maxOutputs: Infinity })).toThrow(RangeError) + expect(() => lookupLimits({ maxTotalBytes: Number.MAX_SAFE_INTEGER + 1 })).toThrow(RangeError) + expect(() => lookupLimits({ maxEvidenceBytes: '8' as unknown as number })).toThrow(RangeError) + }) +}) + +describe('normalizeLookupHost', () => { + it('rejects non-strings, overlong values, credentials, and non-http URLs', () => { + expect(normalizeLookupHost(undefined as unknown as string)).toBeNull() + expect(normalizeLookupHost(`https://example.com/${'a'.repeat(2048)}`)).toBeNull() + expect(normalizeLookupHost('ftp://example.com')).toBeNull() + expect(normalizeLookupHost('https://user@example.com')).toBeNull() + expect(normalizeLookupHost('https://user:pass@example.com')).toBeNull() + expect(normalizeLookupHost('https://:secret@example.com')).toBeNull() + }) + + it('drops query and fragment unless parameters are explicitly allowed', () => { + expect(normalizeLookupHost('https://example.com/?q=1')).toBeNull() + expect(normalizeLookupHost('https://example.com/#frag')).toBeNull() + expect(normalizeLookupHost('https://example.com/?q=1', true)).toBe('https://example.com/?q=1') + expect(normalizeLookupHost('https://example.com/#frag', true)).toBe('https://example.com/#frag') + }) + + it('returns null for values that are not parseable as URLs', () => { + expect(normalizeLookupHost('not a url')).toBeNull() + expect(normalizeLookupHost('https://[')).toBeNull() + expect(normalizeLookupHost('')).toBeNull() + }) +}) + +describe('withLookupAbort', () => { + it('drops an already-aborted waiter and does not leak a rejecting transport', async () => { + const controller = new AbortController() + controller.abort() + const rejected = Promise.reject(new Error('late transport')) + + await expect(withLookupAbort(Promise.resolve('ok'), controller.signal)).rejects.toMatchObject({ + name: 'AbortError', + message: 'Lookup cancelled' + }) + await expect(withLookupAbort(rejected, controller.signal)).rejects.toMatchObject({ + name: 'AbortError' + }) + expect(lookupAbortError().name).toBe('AbortError') + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts new file mode 100644 index 000000000..3608a64ac --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts @@ -0,0 +1,371 @@ +import { readLookupResponseBytes } from '../LookupResponseReader.js' +import { LookupResourceLimitError } from '../LookupResources.js' + +function responseForReader( + reader: ReadableStreamDefaultReader, + contentLength?: string +): Response { + const headers = new Headers() + if (contentLength !== undefined) headers.set('content-length', contentLength) + return { + body: { getReader: () => reader }, + headers + } as unknown as Response +} + +function readerForChunks(chunks: Uint8Array[]): ReadableStreamDefaultReader { + let index = 0 + return { + read: async () => + index < chunks.length + ? { done: false, value: chunks[index++] } + : { done: true, value: undefined }, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader +} + +describe('readLookupResponseBytes', () => { + it('joins normal streamed chunks and charges each accepted chunk', async () => { + const consumed: number[] = [] + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([new Uint8Array([1, 2]), new Uint8Array([3, 4, 5])])), + { maxResponseBytes: 5, consumeBytes: byteCount => consumed.push(byteCount) } + ) + + expect(bytes).toEqual(new Uint8Array([1, 2, 3, 4, 5])) + expect(consumed).toEqual([2, 3]) + }) + + it('allows a body exactly at the configured boundary', async () => { + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([new Uint8Array([1, 2]), new Uint8Array([3])]), '3'), + { maxResponseBytes: 3 } + ) + + expect(bytes).toEqual(new Uint8Array([1, 2, 3])) + }) + + it('rejects a known oversized Content-Length before reading chunks', async () => { + const read = jest.fn() + const cancel = jest.fn(async () => undefined) + const releaseLock = jest.fn() + const reader = { + read, + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader, '6'), { maxResponseBytes: 5 }) + ).rejects.toMatchObject({ name: 'LookupResourceLimitError', limit: 'maxResponseBytes' }) + + expect(read).not.toHaveBeenCalled() + expect(cancel).toHaveBeenCalledTimes(1) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('rejects an overflowing chunk before it is charged or accumulated', async () => { + const cancel = jest.fn(async () => undefined) + const releaseLock = jest.fn() + const reader = { + read: jest + .fn() + .mockResolvedValueOnce({ done: false, value: new Uint8Array([1, 2, 3]) }) + .mockResolvedValueOnce({ done: false, value: new Uint8Array([4, 5, 6]) }), + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + const consumed: number[] = [] + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 5, + consumeBytes: byteCount => consumed.push(byteCount) + }) + ).rejects.toBeInstanceOf(LookupResourceLimitError) + + expect(consumed).toEqual([3]) + expect(cancel).toHaveBeenCalledTimes(1) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('copies producer-owned chunk buffers before the producer reuses them', async () => { + const producerBuffer = new Uint8Array([1, 2]) + let readCount = 0 + const reader = { + read: async () => { + readCount += 1 + if (readCount === 1) return { done: false, value: producerBuffer } + if (readCount === 2) { + producerBuffer.set([3, 4]) + return { done: false, value: producerBuffer } + } + return { done: true, value: undefined } + }, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 4 }) + ).resolves.toEqual(new Uint8Array([1, 2, 3, 4])) + }) + + it('ignores many empty chunks while retaining tiny chunks in bounded storage', async () => { + const emptyChunks = Array.from({ length: 2048 }, () => new Uint8Array(0)) + const tinyChunks = Array.from({ length: 32 }, (_unused, index) => new Uint8Array([index])) + const consumed: number[] = [] + + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([...emptyChunks, ...tinyChunks])), + { maxResponseBytes: 32, consumeBytes: byteCount => consumed.push(byteCount) } + ) + + expect(bytes).toEqual(new Uint8Array(Array.from({ length: 32 }, (_unused, index) => index))) + expect(consumed).toEqual(Array.from({ length: 32 }, () => 1)) + }) + + it('yields so a timer abort can stop an endless eager empty stream', async () => { + const controller = new AbortController() + const cancel = jest.fn(async () => undefined) + const reader = { + read: jest.fn(async () => ({ done: false, value: new Uint8Array(0) })), + cancel, + releaseLock: jest.fn() + } as unknown as ReadableStreamDefaultReader + const timer = setTimeout(() => controller.abort(new Error('empty stream aborted')), 1) + + try { + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 1, + signal: controller.signal + }) + ).rejects.toThrow('empty stream aborted') + } finally { + clearTimeout(timer) + } + + expect(cancel).toHaveBeenCalledTimes(1) + expect((reader.read as jest.Mock).mock.calls.length).toBeGreaterThanOrEqual(64) + expect((reader.read as jest.Mock).mock.calls.length).toBeLessThan(256) + }) + + it('copies the sixty-fourth chunk before yielding to timer-driven producer mutation', async () => { + const producerBuffer = new Uint8Array([1]) + let reads = 0 + const reader = { + read: async () => { + reads += 1 + if (reads <= 64) { + if (reads === 64) setTimeout(() => producerBuffer.fill(9), 0) + return { done: false, value: producerBuffer } + } + return { done: true, value: undefined } + }, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 64 }) + ).resolves.toEqual(new Uint8Array(64).fill(1)) + }) + + it('cleans up when the aggregate byte budget rejects an accepted chunk', async () => { + const cancel = jest.fn(async () => undefined) + const releaseLock = jest.fn() + const reader = { + read: jest.fn().mockResolvedValue({ done: false, value: new Uint8Array([1, 2]) }), + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + const budgetFailure = new Error('aggregate response budget exhausted') + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + consumeBytes: () => { + throw budgetFailure + } + }) + ).rejects.toBe(budgetFailure) + + expect(cancel).toHaveBeenCalledWith(budgetFailure) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('rejects an abort while a reader read remains pending and starts cleanup', async () => { + let rejectRead: (reason?: unknown) => void = () => undefined + const pendingRead = new Promise>((_, reject) => { + rejectRead = reject + }) + const cancel = jest.fn(() => { + rejectRead(new Error('cancelled pending read')) + return Promise.resolve() + }) + const releaseLock = jest.fn() + const reader = { + read: jest.fn(() => pendingRead), + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + const controller = new AbortController() + const aborted = readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + signal: controller.signal + }) + + controller.abort(new Error('lookup aborted')) + + await expect(aborted).rejects.toThrow('lookup aborted') + expect(cancel).toHaveBeenCalledTimes(1) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('preserves the read failure when cancellation and lock release throw', async () => { + const readFailure = new Error('stream failed') + const reader = { + read: jest.fn().mockRejectedValue(readFailure), + cancel: jest.fn(() => { + throw new Error('cancel failed') + }), + releaseLock: jest.fn(() => { + throw new Error('release failed') + }) + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 10 }) + ).rejects.toBe(readFailure) + }) + + it('rejects a maxResponseBytes that is not a non-negative safe integer', async () => { + const response = responseForReader(readerForChunks([])) + await expect( + readLookupResponseBytes(response, { maxResponseBytes: -1 }) + ).rejects.toBeInstanceOf(RangeError) + await expect(readLookupResponseBytes(response, { maxResponseBytes: 1.5 })).rejects.toThrow( + 'maxResponseBytes must be a non-negative safe integer' + ) + await expect( + readLookupResponseBytes(response, { maxResponseBytes: Number.MAX_SAFE_INTEGER + 1 }) + ).rejects.toBeInstanceOf(RangeError) + }) + + it('treats a malformed Content-Length as unknown and still reads the body', async () => { + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([new Uint8Array([9])]), '1e6'), + { maxResponseBytes: 1 } + ) + expect(bytes).toEqual(new Uint8Array([9])) + }) + + it('returns an empty body when the response has no stream', async () => { + const headers = new Headers() + const empty = { body: null, headers } as unknown as Response + await expect(readLookupResponseBytes(empty, { maxResponseBytes: 0 })).resolves.toEqual( + new Uint8Array(0) + ) + + headers.set('content-length', '4') + await expect( + readLookupResponseBytes({ body: null, headers } as unknown as Response, { + maxResponseBytes: 1 + }) + ).rejects.toMatchObject({ name: 'LookupResourceLimitError', limit: 'maxResponseBytes' }) + }) + + it('rejects an already-aborted empty body using the AbortError fallback when no reason is set', async () => { + const signal = { + aborted: true, + reason: undefined, + addEventListener: () => undefined, + removeEventListener: () => undefined + } as unknown as AbortSignal + await expect( + readLookupResponseBytes({ body: null, headers: new Headers() } as unknown as Response, { + maxResponseBytes: 0, + signal + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + + it('rejects an already-aborted stream before the first read', async () => { + const controller = new AbortController() + const read = jest.fn() + controller.abort(new Error('already aborted')) + const reader = { + read, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + signal: controller.signal + }) + ).rejects.toThrow('already aborted') + expect(read).not.toHaveBeenCalled() + }) + + it('treats a missing chunk value as empty input', async () => { + const consumed: number[] = [] + const reader = { + read: jest + .fn() + .mockResolvedValueOnce({ done: false, value: undefined }) + .mockResolvedValueOnce({ done: true, value: undefined }), + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 4, + consumeBytes: byteCount => consumed.push(byteCount) + }) + ).resolves.toEqual(new Uint8Array(0)) + expect(consumed).toEqual([]) + }) + + it('rejects a later read when consumeBytes aborts the signal', async () => { + const controller = new AbortController() + const cancel = jest.fn(async () => undefined) + const reader = { + read: jest + .fn() + .mockResolvedValueOnce({ done: false, value: new Uint8Array([1]) }) + .mockResolvedValueOnce({ done: false, value: new Uint8Array([2]) }), + cancel, + releaseLock: jest.fn() + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + signal: controller.signal, + consumeBytes: () => controller.abort() + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(cancel).toHaveBeenCalledTimes(1) + }) + + it('rejects an abort that races listener registration', async () => { + const signal = { + aborted: false, + reason: new Error('raced abort'), + addEventListener: () => { + signal.aborted = true + }, + removeEventListener: () => undefined + } + await expect( + readLookupResponseBytes(responseForReader(readerForChunks([new Uint8Array([1])])), { + maxResponseBytes: 10, + signal: signal as unknown as AbortSignal + }) + ).rejects.toThrow('raced abort') + }) +}) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9c8f6ca16..b8d5b2bed 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,13 +2,13 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1765000, + "raw": 1787000, "gzip": 430000, "brotli": 330000 }, "esbuild": { - "raw": 1376000, - "gzip": 380000, + "raw": 1394000, + "gzip": 385000, "brotli": 320000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 388fe5344..4ad3142a4 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,14 +2,14 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1817000, - "gzip": 463000, - "brotli": 360000 + "raw": 1839000, + "gzip": 468000, + "brotli": 361500 }, "hermes": { "raw": 3750000, - "gzip": 1538000, - "brotli": 1173000 + "gzip": 1556000, + "brotli": 1188000 } } }