diff --git a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts index b0f5a545dbb3..2821e1ca6def 100644 --- a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts +++ b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts @@ -5,6 +5,7 @@ import type { IIpTableRemoteConfig, IIpTableRuntime, } from '@onekeyhq/shared/src/request/types/ipTable'; +import { pruneIpTableRuntimeSelections } from '@onekeyhq/shared/src/utils/ipTableUtils'; import { SimpleDbEntityBase } from '../base/SimpleDbEntityBase'; @@ -60,20 +61,48 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase { - await this.setRawData((data) => ({ - ...data, - config, - runtime: { - ...(data?.runtime ?? { - enabled: true, - lastUpdated: Date.now(), - lastRegionCheck: 0, - selections: {}, - }), + async saveConfig( + config: IIpTableRemoteConfig, + verifiedMeta?: { payloadHash: string }, + ): Promise { + await this.setRawData((data) => { + const runtime = data?.runtime ?? { + enabled: true, lastUpdated: Date.now(), - }, - })); + lastRegionCheck: 0, + selections: {}, + }; + // The stored config is the signed envelope verbatim, so runtime state + // pointing at endpoints the new config no longer endorses (revoked or + // rotated-out IPs) must be dropped alongside it. + const pruned = pruneIpTableRuntimeSelections({ + config, + selections: runtime.selections, + lastBestIp: runtime.lastBestIp, + }); + return { + ...data, + config, + runtime: { + ...runtime, + selections: pruned.selections, + lastBestIp: pruned.lastBestIp, + lastUpdated: Date.now(), + // Provenance of the last successfully verified config; consumed by + // rollback protection and diagnostics. + ...(verifiedMeta + ? { + lastVerified: { + at: Date.now(), + version: config.version, + generatedAt: config.generated_at, + payloadHash: verifiedMeta.payloadHash, + }, + } + : {}), + }, + }; + }); } /** @@ -105,6 +134,36 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase { + await this.setRawData((data) => { + const runtime = data?.runtime ?? { + enabled: true, + lastUpdated: 0, + lastRegionCheck: 0, + selections: {}, + }; + + return { + ...data, + config: data?.config ?? null, + currentRegion: data?.currentRegion ?? 'AUTO', + runtime: { + ...runtime, + lastBestIp: { + ...runtime.lastBestIp, + [domain]: ip, + }, + }, + version: data?.version ?? 1, + }; + }); + } + @backgroundMethod() async setEnabled(enabled: boolean): Promise { await this.setRawData((data) => ({ diff --git a/packages/kit-bg/src/services/ServiceIpTable.review.test.ts b/packages/kit-bg/src/services/ServiceIpTable.review.test.ts new file mode 100644 index 000000000000..79f717040df4 --- /dev/null +++ b/packages/kit-bg/src/services/ServiceIpTable.review.test.ts @@ -0,0 +1,177 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access */ + +import type { IIpTableRemoteConfig } from '@onekeyhq/shared/src/request/types/ipTable'; + +import ServiceIpTable from './ServiceIpTable'; + +const mockAxiosGet = jest.fn(); + +jest.mock('@onekeyhq/shared/src/background/backgroundDecorators', () => ({ + backgroundClass: () => (target: unknown) => target, + backgroundMethod: + () => (_target: unknown, _key: string, descriptor: PropertyDescriptor) => + descriptor, +})); + +jest.mock('@onekeyhq/shared/src/logger/logger', () => ({ + defaultLogger: { + ipTable: { + request: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, + metrics: { + endpointSwitched: jest.fn(), + configVerifyFailed: jest.fn(), + }, + }, + }, +})); + +jest.mock('@onekeyhq/shared/src/request/helpers/ipTableAdapter', () => ({ + createAxiosWithIpTable: jest.fn(() => ({ get: mockAxiosGet })), + getSelectedIpForHost: Object.assign(jest.fn(), { clear: jest.fn() }), + setReportRequestFailureCallback: jest.fn(), + setReportRequestSuccessCallback: jest.fn(), + testDomainSpeed: jest.fn(), + testIpSpeed: jest.fn(), +})); + +jest.mock('@onekeyhq/shared/src/request/helpers/sniRequest', () => ({ + isProxyActiveForUrl: jest.fn(async () => false), + isSniSupported: jest.fn(() => true), + sniRequest: jest.fn(), +})); + +jest.mock('@onekeyhq/shared/src/request/Interceptor', () => ({ + getRequestHeaders: jest.fn(async () => ({})), +})); + +jest.mock('../states/jotai/atoms', () => ({ + devSettingsPersistAtom: { + get: jest.fn(async () => ({ enabled: false, settings: {} })), + }, +})); + +jest.mock('./ServiceBase', () => ({ + __esModule: true, + default: class ServiceBaseMock { + backgroundApi: any; + + constructor({ backgroundApi }: { backgroundApi: any }) { + this.backgroundApi = backgroundApi; + } + }, +})); + +const DOMAIN = 'onekeycn.com'; + +function buildConfig(ip: string, version = 1): IIpTableRemoteConfig { + return { + version, + ttl_sec: 300, + generated_at: `2026-07-${String(version).padStart(2, '0')}T00:00:00.000Z`, + signature: `signature-${version}`, + domains: { + [DOMAIN]: { + endpoints: [ + { + ip, + provider: 'test', + region: 'ALL', + weight: 1, + }, + ], + }, + }, + }; +} + +function createService() { + const ipTableDb = { + getConfig: jest.fn(), + updateLastBestIp: jest.fn(async () => undefined), + updateSelection: jest.fn(async () => undefined), + }; + const service = new ServiceIpTable({ + backgroundApi: { simpleDb: { ipTable: ipTableDb } }, + }); + return { service, ipTableDb }; +} + +describe('ServiceIpTable review regressions', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + ['empty', null], + ['invalid shape', 'captive portal'], + ])( + 'uses SNI fallback for an HTTP 200 %s CDN response', + async (_name, data) => { + const { service } = createService(); + const fallbackConfig = buildConfig('1.1.1.1'); + mockAxiosGet.mockResolvedValueOnce({ data }); + const fallbackSpy = jest + .spyOn(service as any, 'fetchRemoteConfigViaSniFallback') + .mockResolvedValue(fallbackConfig); + + await expect((service as any).fetchRemoteConfig()).resolves.toEqual( + fallbackConfig, + ); + expect(fallbackSpy).toHaveBeenCalledTimes(1); + }, + ); + + it('discards and queues a rerun when the signed config changes during a speed test', async () => { + const { service, ipTableDb } = createService(); + const originalConfig = buildConfig('1.1.1.1', 1); + // Keep the numeric version unchanged: the signed payload hash and the + // currently endorsed endpoint set must still invalidate the old round. + const replacementConfig = buildConfig('2.2.2.2', 1); + + jest.spyOn(service as any, 'isIpTableEnabled').mockResolvedValue(true); + jest + .spyOn(service as any, 'testMultipleTimes') + .mockResolvedValueOnce(100) + .mockResolvedValueOnce(20); + jest + .spyOn(service, 'getConfig') + .mockResolvedValueOnce({ config: originalConfig, runtime: undefined }) + .mockResolvedValueOnce({ config: replacementConfig, runtime: undefined }); + + await (service as any).selectBestEndpointForDomainInternal(DOMAIN, { + trigger: 'periodic', + }); + + expect(ipTableDb.updateLastBestIp).not.toHaveBeenCalled(); + expect(ipTableDb.updateSelection).not.toHaveBeenCalled(); + expect((service as any).pendingConfigChangeRerun.get(DOMAIN)).toBe( + 'periodic', + ); + }); + + it('starts the queued config-change rerun after the stale round settles', async () => { + const { service } = createService(); + const internalSpy = jest + .spyOn(service as any, 'selectBestEndpointForDomainInternal') + .mockImplementationOnce(async () => { + (service as any).pendingConfigChangeRerun.set(DOMAIN, 'ip_failure'); + }) + .mockResolvedValueOnce(undefined); + + await service.selectBestEndpointForDomain(DOMAIN, { + trigger: 'ip_failure', + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(internalSpy).toHaveBeenCalledTimes(2); + expect(internalSpy).toHaveBeenNthCalledWith(2, DOMAIN, { + trigger: 'ip_failure', + }); + }); +}); diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 567db9578029..96406d2afecd 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -14,6 +14,7 @@ import { } from '@onekeyhq/shared/src/config/appConfig'; import { defaultLogger } from '@onekeyhq/shared/src/logger/logger'; import { + IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, IP_TABLE_INITIAL_SPEED_TEST_DELAY_MS, IP_TABLE_PERFORMANCE_IMPROVEMENT_THRESHOLD, IP_TABLE_SNI_FAILURE_THRESHOLD, @@ -23,21 +24,34 @@ import { IP_TABLE_SPEED_TEST_TIMEOUT_MS, } from '@onekeyhq/shared/src/request/constants/ipTableDefaults'; import { + createAxiosWithIpTable, getSelectedIpForHost, setReportRequestFailureCallback, + setReportRequestSuccessCallback, testDomainSpeed, testIpSpeed, } from '@onekeyhq/shared/src/request/helpers/ipTableAdapter'; -import { isSniSupported } from '@onekeyhq/shared/src/request/helpers/sniRequest'; +import { decideEndpoint } from '@onekeyhq/shared/src/request/helpers/ipTableEndpointDecision'; +import { + applyOutcome, + nextIpTableRequestSequence, +} from '@onekeyhq/shared/src/request/helpers/ipTableOutcomeLedger'; +import { + isProxyActiveForUrl, + isSniSupported, + sniRequest, +} from '@onekeyhq/shared/src/request/helpers/sniRequest'; import { getRequestHeaders } from '@onekeyhq/shared/src/request/Interceptor'; import type { IIpTableConfigWithRuntime, IIpTableRemoteConfig, } from '@onekeyhq/shared/src/request/types/ipTable'; import { + computeIpTableConfigHash, + isIpTableConfigRegression, isSupportIpTablePlatform, - mergeIpTableConfigs, - verifyIpTableConfigSignature, + isValidIpTableRemoteConfigShape, + verifyIpTableConfigSignatureDetailed, } from '@onekeyhq/shared/src/utils/ipTableUtils'; import { devSettingsPersistAtom } from '../states/jotai/atoms'; @@ -54,6 +68,12 @@ interface IEndpointHealth { consecutiveFailures: number; /** Timestamp of last failure */ lastFailureTime: number; + /** + * Runtime-local request sequence of the newest outcome applied to this + * endpoint. Success/failure reports race through async callbacks; an older + * request sequence is stale and ignored even when Date.now() tied. + */ + lastOutcomeSequence: number; } /** @@ -63,8 +83,14 @@ interface IEndpointHealth { interface IDomainHealthStats { /** Health stats for each IP endpoint */ ipEndpoints: Map; - /** Health stats for direct domain connection */ - domainDirect: IEndpointHealth; + /** + * Direct-domain health tracked per hostname (wallet./utility./...): a + * success on one hostname must not clear another hostname's failures, and + * different hostnames' failures must not jointly reach a threshold. Root + * domain routing decisions aggregate over these explicitly. Mirrors the + * adapter-side per-hostname ledger so main and bg runtimes agree. + */ + directHosts: Map; /** Timestamp of last speed test */ lastSpeedTestTime: number; } @@ -126,11 +152,7 @@ class ServiceIpTable extends ServiceBase { if (!stats) { stats = { ipEndpoints: new Map(), - domainDirect: { - failureCount: 0, - consecutiveFailures: 0, - lastFailureTime: 0, - }, + directHosts: new Map(), lastSpeedTestTime: 0, }; this.domainHealthMap.set(domain, stats); @@ -146,22 +168,33 @@ class ServiceIpTable extends ServiceBase { requestType: 'ip' | 'domain', target: string, ): IEndpointHealth { - if (requestType === 'domain') { - return stats.domainDirect; - } - - let endpointHealth = stats.ipEndpoints.get(target); + // For domain requests, target is the hostname; for ip requests it is the + // ip. Both are tracked per target so outcomes of one never contaminate + // another's counters. + const bucket = + requestType === 'domain' ? stats.directHosts : stats.ipEndpoints; + let endpointHealth = bucket.get(target); if (!endpointHealth) { endpointHealth = { failureCount: 0, consecutiveFailures: 0, lastFailureTime: 0, + lastOutcomeSequence: 0, }; - stats.ipEndpoints.set(target, endpointHealth); + bucket.set(target, endpointHealth); } return endpointHealth; } + /** Highest consecutive direct-domain failure count across hostnames */ + private getMaxDirectConsecutiveFailures(stats: IDomainHealthStats): number { + let max = 0; + stats.directHosts.forEach((health) => { + max = Math.max(max, health.consecutiveFailures); + }); + return max; + } + /** * Check if speed test should be triggered based on health stats * Returns true only when current endpoint is failing AND cooldown period has passed @@ -182,16 +215,17 @@ class ServiceIpTable extends ServiceBase { const configWithRuntime = await this.getConfig(); const currentSelection = configWithRuntime.runtime?.selections?.[domain]; - // Check if current active endpoint is failing - if (currentSelection === undefined) { - // No selection yet, don't trigger - return false; - } - - if (currentSelection === '') { - // Using domain directly + // Check if current active endpoint is failing. + // undefined means no selection was ever made — real traffic goes direct + // to the domain in that state, so judge by the domain's health instead of + // never triggering (the old `return false` deadlocked selection forever + // when the initial speed test had not completed). + if (currentSelection === undefined || currentSelection === '') { + // Aggregate across hostnames: any single hostname consistently failing + // is enough to justify a probing round for the root domain. return ( - stats.domainDirect.consecutiveFailures >= IP_TABLE_SNI_FAILURE_THRESHOLD + this.getMaxDirectConsecutiveFailures(stats) >= + IP_TABLE_SNI_FAILURE_THRESHOLD ); } @@ -295,7 +329,13 @@ class ServiceIpTable extends ServiceBase { info: `[IpTable] Fetching remote config from: ${IP_TABLE_CDN_URL}`, }); - const plainAxios = axios.create(); + // Fetch the config through the IP-Table-capable client: the config CDN + // (config.onekeycn.com) shares the walled root domain, so this fetch + // needs the same fail-open protection as business traffic. No cycle: + // endpoint selection only reads the local simpleDb/builtin table. + const plainAxios = createAxiosWithIpTable({ + timeout: IP_TABLE_CDN_FETCH_TIMEOUT_MS, + }); const headers = await getRequestHeaders(); @@ -313,7 +353,17 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.error({ info: '[IpTable] CDN returned empty config', }); - return null; + return this.fetchRemoteConfigViaSniFallback(); + } + + // A misconfigured CDN edge can return HTML or a truncated body that + // still parses; reject it here so the log points at the delivery + // layer instead of a bogus "signature verification failed". + if (!isValidIpTableRemoteConfigShape(remoteConfig)) { + defaultLogger.ipTable.request.error({ + info: '[IpTable] Skipping CDN config update: invalid config shape (delivery-layer problem)', + }); + return this.fetchRemoteConfigViaSniFallback(); } defaultLogger.ipTable.request.info({ @@ -340,12 +390,104 @@ class ServiceIpTable extends ServiceBase { }`, }); } + // The adapter's fail-open needs 3 accumulated transport failures in + // THIS runtime, which a fresh install's single config GET can never + // reach. The config GET is idempotent, so replaying it over SNI with + // known candidate IPs is safe and gives first-run devices on walled + // networks a working config path. + return this.fetchRemoteConfigViaSniFallback(); + } + } + + /** + * Direct SNI fallback for the CDN config GET: selection -> last-best IP -> + * builtin endpoints, first parseable and well-shaped response wins. + */ + private async fetchRemoteConfigViaSniFallback(): Promise { + try { + if (!isSupportIpTablePlatform() || !isSniSupported()) { + return null; + } + let proxyActive: boolean | null = null; + try { + proxyActive = await isProxyActiveForUrl(IP_TABLE_CDN_URL); + } catch { + return null; + } + if (proxyActive === true) { + // A user proxy owns routing; do not bypass it with SNI. + return null; + } + + const url = new URL(IP_TABLE_CDN_URL); + const rootDomain = url.hostname.split('.').slice(-2).join('.'); + const configWithRuntime = await this.getConfig(); + + const candidates: string[] = []; + const selectedIp = configWithRuntime.runtime?.selections?.[rootDomain]; + const lastBestIp = configWithRuntime.runtime?.lastBestIp?.[rootDomain]; + if (selectedIp) { + candidates.push(selectedIp); + } + if (lastBestIp) { + candidates.push(lastBestIp); + } + for (const endpoint of configWithRuntime.config.domains[rootDomain] + ?.endpoints ?? []) { + candidates.push(endpoint.ip); + } + const uniqueCandidates = [...new Set(candidates)].slice(0, 3); + if (uniqueCandidates.length === 0) { + return null; + } + + const headers = await getRequestHeaders(); + for (const ip of uniqueCandidates) { + try { + defaultLogger.ipTable.request.info({ + info: `[IpTable] CDN fetch fallback: trying SNI candidate for ${url.hostname}`, + }); + const response = await sniRequest({ + ip, + hostname: url.hostname, + path: `${url.pathname}${url.search}`, + headers, + method: 'GET', + body: null, + timeout: IP_TABLE_CDN_FETCH_TIMEOUT_MS, + }); + const body = response?.body ?? response?.data; + if (body) { + const parsed: unknown = + typeof body === 'string' ? JSON.parse(body) : body; + if (isValidIpTableRemoteConfigShape(parsed)) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] CDN fetch fallback succeeded via SNI, version: ${parsed.version}`, + }); + return parsed; + } + } + } catch (error) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] CDN fetch fallback candidate failed: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + }); + } + } + return null; + } catch (error) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] CDN fetch fallback error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + }); return null; } } @backgroundMethod() - async fetchAndMergeRemoteConfig(): Promise { + async fetchAndApplyRemoteConfig(): Promise { try { const remoteConfig = await this.fetchRemoteConfig(); @@ -356,11 +498,20 @@ class ServiceIpTable extends ServiceBase { return false; } - const isValidSignature = await verifyIpTableConfigSignature(remoteConfig); + const verifyResult = + await verifyIpTableConfigSignatureDetailed(remoteConfig); - if (!isValidSignature) { + if (!verifyResult.ok) { + // Keep the failure stage in the message: `verifier_load_failed` is a + // packaging/runtime problem (e.g. split-bundle segment refused), + // NOT an untrusted signature — they need different owners. defaultLogger.ipTable.request.error({ - info: '[IpTable] Skipping CDN config update: signature verification failed', + info: `[IpTable] Skipping CDN config update: signature verification failed, reason=${verifyResult.reason}`, + }); + defaultLogger.ipTable.metrics.configVerifyFailed({ + reason: verifyResult.reason, + configVersion: remoteConfig.version, + payloadHash: computeIpTableConfigHash(remoteConfig), }); return false; } @@ -371,16 +522,40 @@ class ServiceIpTable extends ServiceBase { const currentConfig = await this.getConfig(); const localConfig = currentConfig.config; + const lastVerified = currentConfig.runtime?.lastVerified; + + // Rollback protection: refuse configs older than what we already + // verified. Falls back to the active config's own version/generated_at + // as the anchor when no lastVerified exists yet (pre-upgrade installs). + const regressionCheck = isIpTableConfigRegression({ + remoteConfig, + localConfig, + lastVerified, + }); + if (regressionCheck.regression) { + defaultLogger.ipTable.request.error({ + info: `[IpTable] Rejecting CDN config update: ${ + regressionCheck.reason ?? 'regression' + } (remote v${remoteConfig.version}@${remoteConfig.generated_at}, local v${localConfig.version})`, + }); + return false; + } - const mergedConfig = mergeIpTableConfigs(localConfig, remoteConfig); - + // Persist the signed remote envelope VERBATIM. Merging local endpoints + // in would make the effective config diverge from what the signature + // (and the persisted payloadHash) covers, and would make endpoints + // irrevocable: a compromised or rotated-out IP removed from the CDN + // config must actually disappear, including any runtime selection or + // last-best IP that still points at it (pruned inside saveConfig). defaultLogger.ipTable.request.info({ - info: `[IpTable] Merged config has ${ - Object.keys(mergedConfig.domains).length + info: `[IpTable] Applying signed remote config with ${ + Object.keys(remoteConfig.domains).length } domains`, }); - await this.saveConfig(mergedConfig); + await this.backgroundApi.simpleDb.ipTable.saveConfig(remoteConfig, { + payloadHash: computeIpTableConfigHash(remoteConfig), + }); defaultLogger.ipTable.request.info({ info: '[IpTable] CDN config updated successfully', @@ -388,7 +563,7 @@ class ServiceIpTable extends ServiceBase { return true; } catch (error) { defaultLogger.ipTable.request.error({ - info: `[IpTable] Error in fetchAndMergeRemoteConfig: ${ + info: `[IpTable] Error in fetchAndApplyRemoteConfig: ${ error instanceof Error ? error.message : 'Unknown error' }`, }); @@ -484,13 +659,121 @@ class ServiceIpTable extends ServiceBase { } } + // Coalesce concurrent speed tests per domain (anti probe-storm): the log + // of the 2026-07-16 incident shows ~20 concurrent rounds saturating an + // already congested network and polluting the latency measurements. + private speedTestInFlight = new Map< + string, + { + promise: Promise; + trigger: 'domain_failure' | 'ip_failure' | 'periodic'; + } + >(); + + // Monotonic per-domain token bumped on every selection write. A speed test + // captures it at start and must not apply a stale result if someone else + // (e.g. fast failover) wrote a selection while it was probing. + private selectionGeneration = new Map(); + + private bumpSelectionGeneration(domain: string): void { + this.selectionGeneration.set( + domain, + (this.selectionGeneration.get(domain) ?? 0) + 1, + ); + } + + private async applySelection(domain: string, ip: string): Promise { + this.bumpSelectionGeneration(domain); + await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ip); + } + /** * Select best endpoint for a domain - * Compares domain direct connection vs all IPs with SNI - * Prefers domain if IP is not significantly faster (30% threshold) + * Compares domain direct connection vs all IPs with SNI via the pure + * decision policy (reachability-first + hysteresis). When triggered by + * real-traffic domain failures, any reachable IP wins regardless of the + * latency threshold. */ + // Domains that reported real-traffic domain failures while a weaker-trigger + // speed test was in flight; a follow-up domain_failure round runs for them + // so the reachability-first escalation is never silently coalesced away. + private pendingDomainFailureRerun = new Set(); + + // A signed config can change while a probing round is in flight. Keep the + // original trigger so the completed stale round can be discarded and one + // fresh round can run against the newly endorsed endpoint set. + private pendingConfigChangeRerun = new Map< + string, + 'domain_failure' | 'ip_failure' | 'periodic' + >(); + @backgroundMethod() - async selectBestEndpointForDomain(domain: string): Promise { + async selectBestEndpointForDomain( + domain: string, + opts?: { trigger?: 'domain_failure' | 'ip_failure' | 'periodic' }, + ): Promise { + // The kill switch must disable EVERY failover behavior, including the + // reachability-first override that `domain_failure` grants inside + // decideEndpoint (which ignores the 30% threshold). Downgrading the + // trigger here covers the decision override, the stale-write authority + // and the coalescing escalation in one place. + if ( + opts?.trigger === 'domain_failure' && + (await this.isFailoverDisabledByDevSettings()) + ) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Failover disabled by dev settings: downgrading domain_failure trigger to periodic for ${domain}`, + }); + // eslint-disable-next-line no-param-reassign + opts = { trigger: 'periodic' }; + } + const inflight = this.speedTestInFlight.get(domain); + if (inflight) { + // Do not let a stronger trigger be swallowed by a weaker in-flight + // run: the merged run would compute with domainFailingRealTraffic set + // to false and could keep a failing domain selected (the exact + // 2026-07-16 incident shape). Queue one follow-up round instead. + if ( + opts?.trigger === 'domain_failure' && + inflight.trigger !== 'domain_failure' + ) { + this.pendingDomainFailureRerun.add(domain); + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Speed test in flight for ${domain} (trigger=${inflight.trigger}); queueing domain_failure follow-up round`, + }); + } else { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Speed test already in flight for ${domain}, coalescing`, + }); + } + return inflight.promise; + } + const trigger = opts?.trigger ?? 'periodic'; + const promise = this.selectBestEndpointForDomainInternal( + domain, + opts, + ).finally(() => { + this.speedTestInFlight.delete(domain); + const configChangeTrigger = this.pendingConfigChangeRerun.get(domain); + this.pendingConfigChangeRerun.delete(domain); + if (this.pendingDomainFailureRerun.delete(domain)) { + void this.selectBestEndpointForDomain(domain, { + trigger: 'domain_failure', + }); + } else if (configChangeTrigger) { + void this.selectBestEndpointForDomain(domain, { + trigger: configChangeTrigger, + }); + } + }); + this.speedTestInFlight.set(domain, { promise, trigger }); + return promise; + } + + private async selectBestEndpointForDomainInternal( + domain: string, + opts?: { trigger?: 'domain_failure' | 'ip_failure' | 'periodic' }, + ): Promise { // Check if IP Table is enabled if (!(await this.isIpTableEnabled())) { defaultLogger.ipTable.request.info({ @@ -513,6 +796,14 @@ class ServiceIpTable extends ServiceBase { return; } + const startConfigVersion = configWithRuntime.config.version; + const startConfigHash = computeIpTableConfigHash(configWithRuntime.config); + + // A probing round can take tens of seconds; capture the selection + // generation so a stale result cannot overwrite a fast failover that + // happened while we were measuring. + const startGeneration = this.selectionGeneration.get(domain) ?? 0; + try { // 1. Test domain directly defaultLogger.ipTable.request.info({ @@ -554,89 +845,121 @@ class ServiceIpTable extends ServiceBase { }); } - // 3. Find best IP - let bestIp = ''; - let bestIpLatency = Infinity; + // 3. Decide via pure policy (reachability-first + hysteresis) + const { enabled: devSettingEnabled, settings } = + await devSettingsPersistAtom.get(); + const strictMode = Boolean( + devSettingEnabled && settings?.forceIpTableStrict, + ); + const currentSelection = configWithRuntime.runtime?.selections?.[domain]; + const ipLatencies: Record = {}; for (const [ip, latency] of ipResults) { - if (latency < bestIpLatency) { - bestIpLatency = latency; - bestIp = ip; - } + ipLatencies[ip] = latency; } - // 4. Compare and decide - if (domainLatency === Infinity) { - // Domain test failed - if (bestIpLatency !== Infinity) { - // Use best IP - defaultLogger.ipTable.request.info({ - info: `[IpTable] Domain failed, using IP: ${domain} -> ${bestIp}`, - }); - await this.backgroundApi.simpleDb.ipTable.updateSelection( - domain, - bestIp, - ); - } else { - // All tests failed - defaultLogger.ipTable.request.info({ - info: `[IpTable] All tests failed for ${domain}`, - }); - } - return; - } + const decision = decideEndpoint({ + domainLatency, + ipLatencies, + currentSelection, + domainFailingRealTraffic: opts?.trigger === 'domain_failure', + strictMode, + improvementThreshold: IP_TABLE_PERFORMANCE_IMPROVEMENT_THRESHOLD, + }); - if (bestIpLatency === Infinity) { - // All IP tests failed, use domain - defaultLogger.ipTable.request.info({ - info: `[IpTable] All IP tests failed, using domain: ${domain}`, + // Persist the best measured IP for fast failover, regardless of the + // final decision — it is the candidate we switch to when the direct + // domain starts failing real traffic. + const bestMeasuredIp = Object.entries(ipLatencies) + .filter(([, latency]) => latency !== Infinity) + .toSorted((a, b) => a[1] - b[1])[0]?.[0]; + + // The signed config may have been replaced while the probes were + // running. Re-read it immediately before persisting any result: an IP + // removed by the new envelope must never be resurrected as lastBestIp + // or selection by an old round. Queue exactly one fresh round against + // the current config instead. + const latestConfigWithRuntime = await this.getConfig(); + const latestDomainConfig = latestConfigWithRuntime.config.domains[domain]; + const latestConfigHash = computeIpTableConfigHash( + latestConfigWithRuntime.config, + ); + const latestEndpointIps = new Set( + latestDomainConfig?.endpoints.map((endpoint) => endpoint.ip) ?? [], + ); + const measuredEndpointRemoved = [...ipResults.keys()].some( + (ip) => !latestEndpointIps.has(ip), + ); + const selectedCandidateRemoved = + decision.action === 'select_ip' && !latestEndpointIps.has(decision.ip); + if ( + latestConfigWithRuntime.config.version !== startConfigVersion || + latestConfigHash !== startConfigHash || + measuredEndpointRemoved || + selectedCandidateRemoved + ) { + this.pendingConfigChangeRerun.set(domain, opts?.trigger ?? 'periodic'); + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Discarding stale speed test result for ${domain}: signed config changed while probing (v${startConfigVersion}/${startConfigHash} -> v${latestConfigWithRuntime.config.version}/${latestConfigHash}); queueing fresh round`, }); - await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ''); return; } - // Check if forceIpTableStrict mode is enabled - const { enabled: devSettingEnabled, settings } = - await devSettingsPersistAtom.get(); - const forceIpTableStrict = - devSettingEnabled && settings?.forceIpTableStrict; - // In strict mode, always use IP if available (regardless of domain speed) - if (forceIpTableStrict) { - defaultLogger.ipTable.request.info({ - info: `[IpTable] [STRICT MODE] Using best IP: ${domain} -> ${bestIp} (${bestIpLatency}ms)`, - }); - await this.backgroundApi.simpleDb.ipTable.updateSelection( + if (bestMeasuredIp) { + await this.backgroundApi.simpleDb.ipTable.updateLastBestIp( domain, - bestIp, + bestMeasuredIp, ); - return; } - // Normal mode: use threshold-based decision - // Calculate performance improvement - const improvement = (domainLatency - bestIpLatency) / domainLatency; + defaultLogger.ipTable.request.info({ + info: `[IpTable] Endpoint decision for ${domain}: ${decision.action} (${ + decision.reason + }) trigger=${opts?.trigger ?? 'periodic'} domainLatency=${domainLatency}ms`, + }); - if (improvement > IP_TABLE_PERFORMANCE_IMPROVEMENT_THRESHOLD) { - // IP is significantly faster (>30%), use IP - defaultLogger.ipTable.request.info({ - info: `[IpTable] IP is ${(improvement * 100).toFixed( - 1, - )}% faster, using IP: ${domain} -> ${bestIp}`, - }); - await this.backgroundApi.simpleDb.ipTable.updateSelection( - domain, - bestIp, - ); - } else { - // Domain is competitive, prefer domain for stability - defaultLogger.ipTable.request.info({ - info: `[IpTable] Domain is competitive (IP only ${( - improvement * 100 - ).toFixed(1)}% faster), using domain: ${domain}`, + // Compare-and-set: someone (fast failover, another round) wrote a + // selection while this round was probing. A weaker-trigger result + // computed from pre-write state must not clobber it; domain_failure + // results stay authoritative because they encode live traffic reality. + const currentGeneration = this.selectionGeneration.get(domain) ?? 0; + if ( + currentGeneration !== startGeneration && + opts?.trigger !== 'domain_failure' + ) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Discarding stale speed test result for ${domain}: selection changed while probing (trigger=${ + opts?.trigger ?? 'periodic' + })`, }); - await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ''); + return; } + if (decision.action === 'select_ip') { + await this.applySelection(domain, decision.ip); + if ((currentSelection ?? '') === '') { + defaultLogger.ipTable.metrics.endpointSwitched({ + domain, + from: 'domain', + to: 'ip', + trigger: 'speed_test', + reason: decision.reason, + }); + } + } else if (decision.action === 'select_domain') { + await this.applySelection(domain, ''); + if (currentSelection) { + defaultLogger.ipTable.metrics.endpointSwitched({ + domain, + from: 'ip', + to: 'domain', + trigger: 'speed_test', + reason: decision.reason, + }); + } + } + // no_change: keep the previous selection untouched. + // Clean up health stats for unused endpoints after speed test await this.cleanupHealthStatsAfterSpeedTest(domain); } catch (error) { @@ -691,6 +1014,7 @@ class ServiceIpTable extends ServiceBase { domain: string, requestType: 'ip' | 'domain', target: string, + requestSequence?: number, ): Promise { // Check if IP Table is enabled if (!(await this.isIpTableEnabled())) { @@ -698,6 +1022,7 @@ class ServiceIpTable extends ServiceBase { } const now = Date.now(); + const outcomeSequence = requestSequence ?? nextIpTableRequestSequence(); // Get or initialize domain health stats const stats = this.getDomainHealth(domain); @@ -705,9 +1030,24 @@ class ServiceIpTable extends ServiceBase { // Get or initialize endpoint health const endpointHealth = this.getEndpointHealth(stats, requestType, target); - // Update failure statistics + // Outcomes arrive through async callbacks with no ordering guarantee: + // the shared ledger orders them by a monotonic sequence allocated at + // transport hand-off, so a failure whose request predates the newest + // applied outcome never re-increments the counter, even within one ms. + if ( + applyOutcome(endpointHealth, { + ok: false, + requestSequence: outcomeSequence, + }) === 'stale' + ) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Ignoring stale ${requestType} failure for ${domain} (${target}): request predates the newest applied outcome`, + }); + return; + } + + // Update failure statistics (consecutiveFailures is owned by the ledger) endpointHealth.failureCount += 1; - endpointHealth.consecutiveFailures += 1; endpointHealth.lastFailureTime = now; defaultLogger.ipTable.request.warn({ @@ -718,6 +1058,31 @@ class ServiceIpTable extends ServiceBase { }) for ${domain} (${target})`, }); + // Fast failover: after a few consecutive real-request failures on the + // direct domain, switch to the last known-good IP immediately instead of + // waiting for the slow failure threshold + a full speed test round. + if ( + requestType === 'domain' && + endpointHealth.consecutiveFailures === IP_TABLE_DOMAIN_FAILOVER_THRESHOLD + ) { + const configWithRuntime = await this.getConfig(); + const currentSelection = configWithRuntime.runtime?.selections?.[domain]; + const lastBestIp = configWithRuntime.runtime?.lastBestIp?.[domain]; + const failoverDisabled = await this.isFailoverDisabledByDevSettings(); + if (!failoverDisabled && (currentSelection ?? '') === '' && lastBestIp) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Fast failover: domain ${domain} failing real traffic, switching to last-best IP immediately`, + }); + await this.applySelection(domain, lastBestIp); + defaultLogger.ipTable.metrics.endpointSwitched({ + domain, + from: 'domain', + to: 'ip', + trigger: 'fast_failover', + }); + } + } + // Check if speed test should be triggered const shouldTrigger = await this.shouldTriggerSpeedTest(domain, stats); @@ -747,22 +1112,49 @@ class ServiceIpTable extends ServiceBase { stats.lastSpeedTestTime = now; // Reset all consecutive failure counters for this domain (they'll be recounted after speed test) - stats.domainDirect.consecutiveFailures = 0; + stats.directHosts.forEach((health) => { + health.consecutiveFailures = 0; + }); stats.ipEndpoints.forEach((health) => { health.consecutiveFailures = 0; }); // Trigger speed test to find and switch to better endpoint - void this.selectBestEndpointForDomain(domain); + void this.selectBestEndpointForDomain(domain, { + trigger: requestType === 'domain' ? 'domain_failure' : 'ip_failure', + }); + } + + private async isFailoverDisabledByDevSettings(): Promise { + try { + const devSettings = await devSettingsPersistAtom.get(); + // Intentionally ignores devSettings.enabled to match the adapter-side + // check (and the existing disableIpTableInProd convention): main and + // bg runtimes must reach the same conclusion for the same settings. + return Boolean(devSettings.settings?.disableIpTableFailover); + } catch { + // Default to enabled when dev settings are unreadable. + return false; + } } + private pendingInitialSpeedTestTimer: ReturnType | null = + null; + private scheduleSpeedTest(reason: string): void { + if (this.pendingInitialSpeedTestTimer) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] ${reason}, speed test already scheduled, skipping`, + }); + return; + } defaultLogger.ipTable.request.info({ info: `[IpTable] ${reason}, scheduling speed test in ${ IP_TABLE_INITIAL_SPEED_TEST_DELAY_MS / 1000 } s`, }); - setTimeout(() => { + this.pendingInitialSpeedTestTimer = setTimeout(() => { + this.pendingInitialSpeedTestTimer = null; void this.runFullSpeedTest(); }, IP_TABLE_INITIAL_SPEED_TEST_DELAY_MS); } @@ -785,9 +1177,34 @@ class ServiceIpTable extends ServiceBase { }); // Register request failure callback (handles both IP and domain failures) - setReportRequestFailureCallback(({ domain, requestType, target }) => { - void this.reportRequestFailure(domain, requestType, target); - }); + setReportRequestFailureCallback( + ({ domain, requestType, target, requestSequence }) => { + void this.reportRequestFailure( + domain, + requestType, + target, + requestSequence, + ); + }, + ); + + // Successes reset the consecutive-failure counters, so "consecutive" + // means what it says. The success path MUST create the ledger entry when + // none exists yet: a later-started success completing before an + // early-started slow failure has to leave its ordering mark, otherwise + // the old failure would create the entry afterwards and count against a + // link that already proved healthy. + setReportRequestSuccessCallback( + ({ domain, requestType, target, requestSequence }) => { + const stats = this.getDomainHealth(domain); + const endpointHealth = this.getEndpointHealth( + stats, + requestType, + target, + ); + applyOutcome(endpointHealth, { ok: true, requestSequence }); + }, + ); // Try to refresh CDN config if needed const shouldRefresh = await this.shouldRefreshConfig(); @@ -797,7 +1214,7 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.info({ info: '[IpTable] CDN config refresh needed, fetching remote config', }); - const configUpdated = await this.fetchAndMergeRemoteConfig(); + const configUpdated = await this.fetchAndApplyRemoteConfig(); needSpeedTest = true; if (configUpdated) { diff --git a/packages/kit-bg/src/states/jotai/atoms/devSettings.ts b/packages/kit-bg/src/states/jotai/atoms/devSettings.ts index 73c6f797a545..5053b98aa4c0 100644 --- a/packages/kit-bg/src/states/jotai/atoms/devSettings.ts +++ b/packages/kit-bg/src/states/jotai/atoms/devSettings.ts @@ -95,6 +95,9 @@ export interface IDevSettings { // Force IP Table strict mode: always use IP even if runtime.selections is empty // Fallback to first available IP from config when no selection exists forceIpTableStrict?: boolean; + // Kill switch for the fast-failover behaviors introduced for extreme + // network conditions (adapter fail-open + service fast switch to last-best IP) + disableIpTableFailover?: boolean; // Enable mock market banner data for UI testing enableMockMarketBanner?: boolean; // Test accounts for OneKey ID login testing @@ -175,6 +178,7 @@ export const { usbCommunicationMode: 'webusb', disableIpTableInProd: false, // IP Table enabled by default forceIpTableStrict: false, // Strict mode: disabled by default + disableIpTableFailover: false, // Fast failover enabled by default useFastPbkdf2NativeBackend: false, }, }, diff --git a/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx b/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx index 7ac961287389..5d42661ed739 100644 --- a/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx +++ b/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx @@ -1300,6 +1300,18 @@ const BaseDevSettingsSection = () => { > + + + ({ warn: jest.fn(), error: jest.fn(), }, + metrics: { + endpointSwitched: jest.fn(), + adapterFailover: jest.fn(), + configVerifyFailed: jest.fn(), + }, }, }, })); @@ -228,3 +239,774 @@ describe('ipTableAdapter SNI preflight and fail-closed behavior', () => { ); }); }); + +describe('ipTableAdapter fail-open on domain network failures', () => { + const BUILTIN_CN_IPS = [ + '104.18.20.233', + '104.18.21.233', + '216.19.3.115', + '216.19.2.116', + '216.19.4.106', + ]; + + let originalAdapter: typeof axios.defaults.adapter; + let fallbackAdapter: jest.Mock< + Promise, + [InternalAxiosRequestConfig] + >; + let consoleErrorSpy: jest.SpyInstance; + + function networkError() { + // Transport-level failure: no HTTP response attached. + return Object.assign(new Error('timeout of 30000ms exceeded'), { + code: 'ECONNABORTED', + }); + } + + function httpError(status: number) { + return Object.assign(new Error(`Request failed with status ${status}`), { + response: { status }, + }); + } + + beforeEach(() => { + consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + originalAdapter = axios.defaults.adapter; + fallbackAdapter = jest.fn(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + axios.defaults.adapter = fallbackAdapter; + + mockedIsSniSupported.mockReturnValue(true); + mockedIsProxyActiveForUrl.mockResolvedValue(false); + mockedSniRequest.mockReset(); + mockedGetRequestHeaders.mockResolvedValue({}); + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + settings: {}, + } as never); + // Simulate the main runtime / cold-start window: no config installed. + mockedRequestHelper.getIpTableConfig.mockResolvedValue(null as never); + resetAdapterFailoverStatesForTesting(); + }); + + afterEach(() => { + axios.defaults.adapter = originalAdapter; + consoleErrorSpy.mockRestore(); + resetAdapterFailoverStatesForTesting(); + jest.restoreAllMocks(); + jest.clearAllMocks(); + }); + + async function failNTimes(n: number) { + fallbackAdapter.mockImplementation(async () => { + throw networkError(); + }); + for (let i = 0; i < n; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + } + + test('activates fail-open after 3 consecutive network errors and routes next request via builtin ip', async () => { + await failNTimes(3); + expect(mockedSniRequest).not.toHaveBeenCalled(); + + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ status: 200, data: { ok: true } }); + + expect(mockedSniRequest).toHaveBeenCalledTimes(1); + const sniArgs = mockedSniRequest.mock.calls[0][0] as { + ip: string; + hostname: string; + }; + expect(BUILTIN_CN_IPS).toContain(sniArgs.ip); + expect(sniArgs.hostname).toBe('wallet.onekeycn.com'); + }); + + test('does not replay the failing request itself', async () => { + await failNTimes(3); + // Every one of the 3 failing requests must reject; no hidden retry via SNI. + expect(fallbackAdapter).toHaveBeenCalledTimes(3); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('http error responses do not count as network failures', async () => { + fallbackAdapter.mockImplementation(async () => { + throw httpError(500); + }); + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ response: { status: 500 } }); + } + + // Next request still goes direct domain: fail-open must not be active. + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('an http error response resets the consecutive failure counter', async () => { + // 2 transport failures, then a 500 response, then 2 more transport + // failures: without the reset this would be 4 "consecutive" failures + // and the circuit would open despite proof the path works. + await failNTimes(2); + fallbackAdapter.mockImplementation(async () => { + throw httpError(500); + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ response: { status: 500 } }); + await failNTimes(2); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('a success resets the consecutive failure counter', async () => { + await failNTimes(2); + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ); + await failNTimes(2); + + // 2 + reset + 2 consecutive failures: threshold (3) not reached. + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('fail-open expires after its TTL window', async () => { + await failNTimes(3); + + const realNow = Date.now(); + const nowSpy = jest + .spyOn(Date, 'now') + .mockReturnValue(realNow + 5 * 60_000 + 1); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + + nowSpy.mockRestore(); + }); + + test('kill switch disables fail-open entirely', async () => { + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + enabled: true, + settings: { disableIpTableFailover: true }, + } as never); + + await failNTimes(3); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('flipping the kill switch while fail-open is active takes effect immediately', async () => { + await failNTimes(3); + + // Circuit is open now; turn the kill switch on afterwards. + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + enabled: true, + settings: { disableIpTableFailover: true }, + } as never); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('cancellations do not count as transport failures', async () => { + fallbackAdapter.mockImplementation(async () => { + throw Object.assign(new Error('canceled'), { code: 'ERR_CANCELED' }); + }); + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ERR_CANCELED' }); + } + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('errors without an allowlisted transport code do not count', async () => { + fallbackAdapter.mockImplementation(async () => { + throw Object.assign( + new OneKeyLocalError('something exploded internally'), + { + code: 'SOME_INTERNAL_ERROR', + }, + ); + }); + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ message: 'something exploded internally' }); + } + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('a domain success on another hostname does not close the circuit', async () => { + // wallet.* opens the circuit. + await failNTimes(3); + + // While the circuit is open, utility.* goes via SNI too; make its SNI + // attempt fail ambiguously (GET is idempotent -> falls back to domain) + // and let the domain succeed. That domain success comes from a hostname + // that did NOT open the circuit, so the circuit must stay open. + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + mockedSniRequest.mockRejectedValueOnce( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildConfig('https://utility.onekeycn.com/utility/v1/something'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + + // wallet traffic must still route via SNI (circuit still open). + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ status: 200, data: { ok: true } }); + const lastSniCall = mockedSniRequest.mock.calls.at(-1)?.[0] as { + hostname: string; + }; + expect(lastSniCall.hostname).toBe('wallet.onekeycn.com'); + }); + + test('a late failure from a request started before a newer success does not count', async () => { + // Both transports start in the same wall-clock millisecond. Ordering must + // come from the runtime request sequence, not Date.now(). + jest.spyOn(Date, 'now').mockReturnValue(1000); + // Request A starts first and fails slowly; request B starts later and + // succeeds before A's failure lands. B's success must leave its ledger + // mark even though no failure entry existed yet, making A's late + // failure stale — otherwise a recovered hostname would accumulate + // toward fail-open. + let releaseSlowFail: (() => void) | undefined; + const slowFailGate = new Promise((resolve) => { + releaseSlowFail = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('slow-fail')) { + await slowFailGate; + throw networkError(); + } + if (config.url?.includes('ok-probe')) { + return { + data: { probe: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + const slowFailing = createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/slow-fail'), + ); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // Later-started success completes first. + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + + // Now the early-started failure lands: it must be ignored as stale. + releaseSlowFail?.(); + await expect(slowFailing).rejects.toMatchObject({ code: 'ECONNABORTED' }); + + // Two fresh failures: total applied failures is 2, not 3. + for (let i = 0; i < 2; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // The circuit must still be closed: the next request goes via domain. + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('after recovery closes the circuit, old failures landing later stay stale', async () => { + // Three slow requests start BEFORE anything goes wrong. The circuit then + // opens, recovers (which must preserve the per-hostname watermark), and + // only afterwards do the three old failures land: they must all be + // stale — if deactivation had cleared the ledger they would re-open the + // circuit on a link that already proved healthy. + let releaseOldFailures: (() => void) | undefined; + const oldFailureGate = new Promise((resolve) => { + releaseOldFailures = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('old-slow-fail')) { + await oldFailureGate; + throw networkError(); + } + if (config.url?.includes('ok-probe')) { + return { + data: { probe: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + const oldSlowFailures = [1, 2, 3].map((i) => + createIpTableAdapter({})( + buildConfig(`https://wallet.onekeycn.com/wallet/v1/old-slow-fail-${i}`), + ), + ); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // Open the circuit with 3 fresh transport failures. + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // Recover: circuit is open, so the probe goes SNI first; an ambiguous + // timeout on a GET falls back to the domain, and that domain success + // (activated hostname, started after activation) closes the circuit. + mockedSniRequest.mockRejectedValueOnce( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + + // Old failures land only now — after recovery. + releaseOldFailures?.(); + for (const pending of oldSlowFailures) { + await expect(pending).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // Circuit must still be closed: next request goes via domain, not SNI. + mockedSniRequest.mockClear(); + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('a late success from a request started before activation does not close the circuit', async () => { + // One shared implementation routed by URL marker so the in-flight stale + // request is unaffected when the failing behavior is exercised. + let resolveStale: (() => void) | undefined; + const stalePending = new Promise((resolve) => { + resolveStale = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('stale-probe')) { + await stalePending; + return { + data: { stale: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + const staleRequest = createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/stale-probe'), + ); + // Ensure the stale request's transport start timestamp precedes the + // activation timestamp deterministically. + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // Open the circuit with 3 fresh transport failures. + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // Let the stale request resolve successfully AFTER activation. + resolveStale?.(); + await expect(staleRequest).resolves.toMatchObject({ + data: { stale: true }, + }); + + // Circuit must still be open: next request goes via SNI. + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ status: 200, data: { ok: true } }); + expect(mockedSniRequest).toHaveBeenCalledTimes(1); + }); +}); + +describe('ipTableAdapter idempotency-gated fallback after SNI started', () => { + let originalAdapter: typeof axios.defaults.adapter; + let fallbackAdapter: jest.Mock< + Promise, + [InternalAxiosRequestConfig] + >; + let consoleErrorSpy: jest.SpyInstance; + + function buildMethodConfig( + url: string, + method: string, + ): InternalAxiosRequestConfig { + return { + url, + method, + headers: axios.AxiosHeaders.from({}), + } as InternalAxiosRequestConfig; + } + + beforeEach(() => { + consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + originalAdapter = axios.defaults.adapter; + fallbackAdapter = jest.fn(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + axios.defaults.adapter = fallbackAdapter; + + mockedIsSniSupported.mockReturnValue(true); + mockedIsProxyActiveForUrl.mockResolvedValue(false); + mockedSniRequest.mockReset(); + mockedGetRequestHeaders.mockResolvedValue({}); + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + settings: {}, + } as never); + mockedRequestHelper.getIpTableConfig.mockResolvedValue({ + config: { + version: 1, + ttl_sec: 60, + generated_at: '2026-06-30T00:00:00.000Z', + signature: '', + domains: { + 'example.com': { + endpoints: [ + { + ip: '93.184.216.34', + provider: 'test', + region: 'ALL', + weight: 1, + }, + ], + }, + }, + }, + runtime: { + enabled: true, + lastUpdated: 0, + lastRegionCheck: 0, + selections: { + 'example.com': '93.184.216.34', + }, + }, + } as never); + resetAdapterFailoverStatesForTesting(); + }); + + afterEach(() => { + axios.defaults.adapter = originalAdapter; + consoleErrorSpy.mockRestore(); + resetAdapterFailoverStatesForTesting(); + jest.clearAllMocks(); + }); + + test('GET falls back to domain after an ambiguous SNI timeout', async () => { + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'get'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(fallbackAdapter).toHaveBeenCalledTimes(1); + }); + + test('POST does NOT fall back after an ambiguous SNI timeout (double-send risk)', async () => { + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ code: 'SNI_TIMEOUT' }); + expect(fallbackAdapter).not.toHaveBeenCalled(); + }); + + test('POST falls back when the error proves the connection was never established', async () => { + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('connection refused'), { + code: 'SNI_CONNECTION_REFUSED', + }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(fallbackAdapter).toHaveBeenCalledTimes(1); + }); + + test('POST does NOT fall back on SNI_NETWORK_UNREACHABLE (iOS maps connection-lost to it)', async () => { + // NSURLErrorNetworkConnectionLost — which can fire after the body was + // sent — maps to SNI_NETWORK_UNREACHABLE on iOS, so this code cannot + // prove the request was never written. + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('network unreachable'), { + code: 'SNI_NETWORK_UNREACHABLE', + }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ code: 'SNI_NETWORK_UNREACHABLE' }); + expect(fallbackAdapter).not.toHaveBeenCalled(); + }); + + test('POST does NOT fall back when SNI returns a null response', async () => { + mockedSniRequest.mockResolvedValue(null); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ + message: expect.stringContaining('not idempotent'), + }); + expect(fallbackAdapter).not.toHaveBeenCalled(); + }); + + test('one sni attempt reports at most one ip failure (null response + blocked fallback)', async () => { + // The null-response branch reports an ip failure, then throws for + // non-idempotent requests; the outer catch must not report the same + // failure a second time. + const failureSpy = jest.fn(); + setReportRequestFailureCallback(failureSpy); + try { + mockedSniRequest.mockResolvedValue(null); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ + message: expect.stringContaining('not idempotent'), + }); + expect(failureSpy).toHaveBeenCalledTimes(1); + expect(failureSpy).toHaveBeenCalledWith( + expect.objectContaining({ + requestType: 'ip', + target: '93.184.216.34', + requestSequence: expect.any(Number), + }), + ); + } finally { + setReportRequestFailureCallback(() => undefined); + } + }); +}); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 5075ff71f07b..5d56a7bb7027 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -4,13 +4,24 @@ import { OneKeyLocalError } from '../../errors'; import { defaultLogger } from '../../logger/logger'; import platformEnv from '../../platformEnv'; import { memoizee } from '../../utils/cacheUtils'; +import { + DEFAULT_IP_TABLE_CONFIG, + IP_TABLE_ADAPTER_FAILOVER_TTL_MS, + IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, +} from '../constants/ipTableDefaults'; import { getRequestHeaders } from '../Interceptor'; import requestHelper from '../requestHelper'; +import { + applyOutcome, + createOutcomeLedgerEntry, + nextIpTableRequestSequence, +} from './ipTableOutcomeLedger'; import { isSniFailClosedError } from './sniFailClosedError'; import { redactIpLiterals, safeSniLogValue } from './sniLogRedaction'; import { isProxyActiveForUrl, isSniSupported, sniRequest } from './sniRequest'; +import type { IIpTableOutcomeLedgerEntry } from './ipTableOutcomeLedger'; import type { AxiosAdapter, AxiosRequestConfig, @@ -100,12 +111,67 @@ interface IRequestFailureParams { target: string; /** Error message */ error: string; + /** Runtime-local sequence allocated when handed to the transport */ + requestSequence: number; } let reportRequestFailureCallback: | ((params: IRequestFailureParams) => void) | null = null; +/** + * Request success callback parameters. Successes must flow to the service so + * its consecutive-failure counters reset on real recovery instead of only + * ever incrementing. + */ +interface IRequestSuccessParams { + domain: string; + requestType: 'ip' | 'domain'; + target: string; + /** Runtime-local sequence allocated when handed to the transport */ + requestSequence: number; +} + +let reportRequestSuccessCallback: + | ((params: IRequestSuccessParams) => void) + | null = null; + +export function setReportRequestSuccessCallback( + callback: (params: IRequestSuccessParams) => void, +) { + reportRequestSuccessCallback = callback; +} + +/** + * Once a request has been handed to the SNI transport it may have reached + * the server even when we got no usable response back. Re-sending it over + * the domain is only safe for idempotent methods, or when the error code + * proves the connection was never established (nothing was written). + */ +const IDEMPOTENT_HTTP_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']); + +// SNI_NETWORK_UNREACHABLE is deliberately NOT in this set: on iOS the native +// module maps NSURLErrorNetworkConnectionLost — which can fire after the body +// was fully sent and the server executed the request — to that code +// (SniConnectClient.swift maps NotConnectedToInternet and +// NetworkConnectionLost to the same .networkUnreachable case), so it cannot +// prove the request was never written. +const SNI_PRE_WRITE_ERROR_CODES = new Set([ + 'SNI_CONNECTION_REFUSED', + 'SNI_DNS_FAILED', + 'SNI_INVALID_URL', +]); + +function canFallbackAfterSniStarted(method: string, error?: unknown): boolean { + if (IDEMPOTENT_HTTP_METHODS.has(method.toUpperCase())) { + return true; + } + if (error && SNI_PRE_WRITE_ERROR_CODES.has(getErrorCode(error))) { + return true; + } + return false; +} + /** * Extract root domain from hostname * Example: wallet.example.com -> example.com @@ -149,6 +215,263 @@ async function getMappedDomainForIpLookup( } } +// getSelectedIpForHostInternal is a hoisted function declaration, so wiring +// the memoized wrapper up here (before the fail-open helpers that clear its +// cache) is safe at module-evaluation time. +const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { + promise: true, + maxAge: 5000, // 5 seconds cache + max: 100, // Max 100 hostname cached + primitive: true, // hostname is a string primitive, use simple equality check +}); + +// ========== Fail-open on domain network failures ========== +// +// When real requests on the direct domain keep failing at the transport +// level (timeouts, resets — NOT HTTP error responses or cancellations), the +// adapter opens a time-boxed window during which hosts of that root domain +// resolve to a fallback IP (runtime last-best IP when available, builtin +// config otherwise). This is the only protection available in runtimes where +// the simpleDb-backed config is absent: the main runtime on split-runtime +// targets and the cold-start window before background init completes. +// The request that trips the threshold is never replayed — only subsequent +// requests take the fail-open route — and once a request has been handed to +// the SNI transport, falling back to the domain (which re-sends it) is +// restricted to idempotent methods or provably-unsent errors; see +// canFallbackAfterSniStarted. + +interface IAdapterFailoverState { + /** + * Per-hostname outcome ledger (consecutive transport failures + newest + * applied outcome ordered by runtime request sequence). Entries persist across + * successes so a late-arriving old failure can always be recognized as + * stale — deleting them would lose the ordering mark. + */ + hostFailures: Map; + failOpenUntil: number; + /** Sequence of the failure request that opened the circuit */ + activatedRequestSequence: number; + /** Hostnames whose failures opened the circuit; only they may close it early */ + activatedHostnames: Set; + /** Rotates across activations so a dead builtin endpoint is not retried forever */ + fallbackIpIndex: number; +} + +function getHostOutcomeLedger( + state: IAdapterFailoverState, + hostname: string, +): IIpTableOutcomeLedgerEntry { + let entry = state.hostFailures.get(hostname); + if (!entry) { + entry = createOutcomeLedgerEntry(); + state.hostFailures.set(hostname, entry); + } + return entry; +} + +const adapterFailoverStates = new Map(); + +/** Test-only helper: clears fail-open state and the selection memo cache. */ +export function resetAdapterFailoverStatesForTesting(): void { + adapterFailoverStates.clear(); + getSelectedIpForHost.clear(); +} + +/** + * Transport-level failure allowlist. Only errors that prove the network path + * itself is broken may open the circuit. Cancellations (AbortController, + * axios dedup, page unload) and HTTP error responses must never count: a + * cancel says nothing about the network, and a response proves it works. + */ +const TRANSPORT_ERROR_CODES = new Set([ + 'ERR_NETWORK', // axios v1 generic network failure (xhr/fetch) + 'ECONNABORTED', // axios timeout + 'ETIMEDOUT', + 'ECONNRESET', + 'ECONNREFUSED', + 'ENOTFOUND', + 'ENETUNREACH', + 'EHOSTUNREACH', + 'EAI_AGAIN', +]); + +function isTransportLevelError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + if (axios.isCancel(error)) { + return false; + } + if ('response' in error && (error as { response?: unknown }).response) { + return false; + } + const code = getErrorCode(error); + return TRANSPORT_ERROR_CODES.has(code); +} + +async function isFailoverDisabledByDevSettings(): Promise { + try { + const devSettings = await requestHelper.getDevSettingsPersistAtom(); + return !!devSettings.settings?.disableIpTableFailover; + } catch { + // Default to enabled when dev settings are unreadable. + return false; + } +} + +function getFailoverState(lookupDomain: string): IAdapterFailoverState { + let state = adapterFailoverStates.get(lookupDomain); + if (!state) { + state = { + hostFailures: new Map(), + failOpenUntil: 0, + activatedRequestSequence: 0, + activatedHostnames: new Set(), + fallbackIpIndex: 0, + }; + adapterFailoverStates.set(lookupDomain, state); + } + return state; +} + +function isFailoverActive(lookupDomain: string): boolean { + const state = adapterFailoverStates.get(lookupDomain); + return Boolean(state && state.failOpenUntil > Date.now()); +} + +function deactivateFailover(lookupDomain: string, cause: string): void { + const state = adapterFailoverStates.get(lookupDomain); + if (!state) { + return; + } + if (state.failOpenUntil > Date.now()) { + logIpTableEvent('info', 'adapter_failover_deactivated', { + lookupDomain, + cause, + }); + defaultLogger.ipTable.metrics.adapterFailover({ + lookupDomain, + action: 'deactivated', + }); + } + // Reset counters but PRESERVE each hostname's lastOutcomeSequence watermark: + // clearing entries would let failures from requests started before the + // recovery re-apply on fresh entries and reopen a circuit that the link + // already proved healthy. + state.hostFailures.forEach((entry) => { + entry.consecutiveFailures = 0; + }); + state.failOpenUntil = 0; + state.activatedRequestSequence = 0; + state.activatedHostnames.clear(); + getSelectedIpForHost.clear(); +} + +async function resolveFailoverLookupDomain( + rootDomain: string, +): Promise { + const mappedDomain = await getMappedDomainForIpLookup(rootDomain); + return mappedDomain || rootDomain; +} + +async function recordDomainRequestOutcome(options: { + rootDomain: string; + hostname: string; + ok: boolean; + /** Runtime-local sequence allocated when handed to the transport */ + requestSequence: number; + error?: unknown; +}): Promise { + const { rootDomain, hostname, ok, requestSequence, error } = options; + const lookupDomain = await resolveFailoverLookupDomain(rootDomain); + const state = getFailoverState(lookupDomain); + + // Both success and failure go through the same request-sequence-ordered + // ledger: a success records its order even on a fresh entry (so an older + // failure arriving later is recognized as stale), and a failure whose + // request predates the newest applied outcome never counts. + const ledger = getHostOutcomeLedger(state, hostname); + + if (ok) { + const applied = applyOutcome(ledger, { ok: true, requestSequence }); + // A success only proves the path that was exercised: it may close the + // circuit only when it comes from a hostname that opened it AND started + // after activation (a long-in-flight request resolving late says + // nothing about recovery). + if ( + applied === 'applied' && + state.failOpenUntil > Date.now() && + state.activatedHostnames.has(hostname) && + requestSequence > state.activatedRequestSequence + ) { + deactivateFailover(lookupDomain, 'domain_recovered'); + } + return; + } + + if (!isTransportLevelError(error)) { + return; + } + if (await isFailoverDisabledByDevSettings()) { + return; + } + + if (applyOutcome(ledger, { ok: false, requestSequence }) === 'stale') { + return; + } + + if (ledger.consecutiveFailures >= IP_TABLE_DOMAIN_FAILOVER_THRESHOLD) { + const now = Date.now(); + state.activatedHostnames.add(hostname); + if (state.failOpenUntil <= now) { + state.failOpenUntil = now + IP_TABLE_ADAPTER_FAILOVER_TTL_MS; + state.activatedRequestSequence = requestSequence; + state.fallbackIpIndex += 1; + getSelectedIpForHost.clear(); + logIpTableEvent('warn', 'adapter_failover_activated', { + lookupDomain, + hostname, + consecutiveNetworkFailures: ledger.consecutiveFailures, + ttlMs: IP_TABLE_ADAPTER_FAILOVER_TTL_MS, + }); + defaultLogger.ipTable.metrics.adapterFailover({ + lookupDomain, + action: 'activated', + }); + } + } +} + +function getFailoverFallbackIp(lookupDomain: string): string | null { + const endpoints = DEFAULT_IP_TABLE_CONFIG.domains[lookupDomain]?.endpoints; + if (!endpoints || endpoints.length === 0) { + return null; + } + const state = getFailoverState(lookupDomain); + return endpoints[state.fallbackIpIndex % endpoints.length].ip; +} + +/** + * Resolve the fail-open IP for a lookup domain when the circuit is open. + * Prefers the last-best IP measured by the background speed test when a + * runtime is available. Honors the kill switch immediately: flipping + * `disableIpTableFailover` while a circuit is open clears the state instead + * of waiting for the TTL to expire. + */ +async function getActiveFailoverIp( + lookupDomain: string, + runtimeLastBestIp: string | undefined, +): Promise { + if (!isFailoverActive(lookupDomain)) { + return null; + } + if (await isFailoverDisabledByDevSettings()) { + deactivateFailover(lookupDomain, 'kill_switch'); + return null; + } + return runtimeLastBestIp || getFailoverFallbackIp(lookupDomain); +} + /** * Check if IP Table should be used based on environment and dev settings * @returns true if IP Table should be used, false otherwise @@ -216,6 +539,27 @@ async function getSelectedIpForHostInternal( // Check if config exists and is enabled if (!configWithRuntime || configWithRuntime.runtime?.enabled === false) { + // No config = the main runtime or the cold-start window before the + // background init completes. When the domain is failing at transport + // level, fail-open to a builtin IP so those runtimes have protection + // too. runtime.enabled === false is explicit user intent: no fail-open. + if (!configWithRuntime) { + const lookupDomain = await resolveFailoverLookupDomain(rootDomain); + const failoverIp = await getActiveFailoverIp(lookupDomain, undefined); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: lookupDomain !== rootDomain, + strictMode: false, + runtimeEnabled: false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; + } + } logIpTableEvent('info', 'iptable_selection', { hostname, rootDomain, @@ -269,6 +613,23 @@ async function getSelectedIpForHostInternal( // In strict mode, override this and use fallback IP from config if (selectedIp === '') { if (!strictMode) { + const failoverIp = await getActiveFailoverIp( + lookupDomain, + runtime?.lastBestIp?.[lookupDomain], + ); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: Boolean(mappedDomain), + strictMode: false, + runtimeEnabled: runtime?.enabled !== false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; + } debugLog( `[IpTableAdapter] Explicitly using domain for: ${lookupDomain}`, ); @@ -308,6 +669,24 @@ async function getSelectedIpForHostInternal( } } + const noSelectionFailoverIp = await getActiveFailoverIp( + lookupDomain, + runtime?.lastBestIp?.[lookupDomain], + ); + if (noSelectionFailoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: Boolean(mappedDomain), + strictMode: Boolean(strictMode), + runtimeEnabled: runtime?.enabled !== false, + decision: 'fail_open', + selectedIpHash: hashForLog(noSelectionFailoverIp), + }); + return noSelectionFailoverIp; + } + logIpTableEvent('info', 'iptable_selection', { hostname, rootDomain, @@ -344,13 +723,6 @@ async function getSelectedIpForHostInternal( } } -const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { - promise: true, - maxAge: 5000, // 5 seconds cache - max: 100, // Max 100 hostname cached - primitive: true, // hostname is a string primitive, use simple equality check -}); - /** * Convert AxiosHeaders to plain object */ @@ -465,6 +837,7 @@ export function createIpTableAdapter( rootDomain?: string; }): Promise => { const { config, isFallback = false, hostname, rootDomain } = options; + const requestSequence = nextIpTableRequestSequence(); debugLog('[IpTableAdapter] About to call original adapter...'); debugLog( '[IpTableAdapter] Original adapter type:', @@ -521,14 +894,80 @@ export function createIpTableAdapter( ); } + if (rootDomain && hostname) { + // A completed domain request (any HTTP status) proves the network + // path works for THIS hostname: reset its fail-open counter and + // notify the service so its health stats reset too. + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: true, + requestSequence, + }); + if (reportRequestSuccessCallback) { + reportRequestSuccessCallback({ + domain: rootDomain, + requestType: 'domain', + target: hostname, + requestSequence, + }); + } + } + return response; } catch (error) { - // Only report domain failures if this is NOT a fallback request + // A rejected promise carrying an HTTP response (validateStatus said + // no to a 4xx/5xx) still proves the transport path works: reset the + // counters exactly like a success before surfacing the error, so + // transport failures separated by healthy responses can never + // accumulate to the fail-open threshold. + const httpResponseReceived = Boolean( + error && + typeof error === 'object' && + 'response' in error && + (error as { response?: unknown }).response, + ); + + if (rootDomain && hostname) { + if (httpResponseReceived) { + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: true, + requestSequence, + }); + if (reportRequestSuccessCallback) { + reportRequestSuccessCallback({ + domain: rootDomain, + requestType: 'domain', + target: hostname, + requestSequence, + }); + } + } else { + // Track transport-level failures for adapter fail-open. The + // failing request itself is never replayed (no double-send of + // POSTs); only subsequent requests take the fail-open route. + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: false, + requestSequence, + error, + }); + } + } + + // Only report domain failures if this is NOT a fallback request, and + // only for transport-level failures: HTTP responses prove the network + // path works and cancellations say nothing about it, so neither may + // drive the service's failover/speed-test counters. if ( !isFallback && hostname && rootDomain && - reportRequestFailureCallback + reportRequestFailureCallback && + isTransportLevelError(error) ) { debugLog( `[IpTableAdapter] Domain request failed (not fallback): ${hostname}`, @@ -538,6 +977,7 @@ export function createIpTableAdapter( requestType: 'domain', target: hostname, error: error instanceof Error ? error.message : String(error), + requestSequence, }); } @@ -783,6 +1223,26 @@ export function createIpTableAdapter( requestBody ? requestBody.substring(0, 200) : 'null', ); + const sniRequestSequence = nextIpTableRequestSequence(); + // One SNI attempt must produce at most one ip-failure report: the + // null-response branch throws for non-idempotent requests and that + // throw lands in the same catch below, which would otherwise report the + // same failure a second time. + let ipFailureReported = false; + const reportIpFailureOnce = (errorMessage: string) => { + if (ipFailureReported || !reportRequestFailureCallback) { + return; + } + ipFailureReported = true; + reportRequestFailureCallback({ + domain: rootDomain, + requestType: 'ip', + target: selectedIp, + error: errorMessage, + requestSequence: sniRequestSequence, + }); + }; + try { const sniResponse = await sniRequest({ ip: selectedIp, @@ -797,14 +1257,21 @@ export function createIpTableAdapter( // If SNI request fails, use original adapter if (!sniResponse) { debugLog('[IpTableAdapter] SNI request returned null, using fallback'); - // Report IP failure - if (reportRequestFailureCallback) { - reportRequestFailureCallback({ - domain: rootDomain, - requestType: 'ip', - target: selectedIp, - error: 'SNI response null', + reportIpFailureOnce('SNI response null'); + // A null response is ambiguous — the request may have reached the + // server. Re-sending over the domain is only safe for idempotent + // methods; anything else must surface the failure to the caller. + const method = (config.method || 'GET').toUpperCase(); + if (!canFallbackAfterSniStarted(method)) { + logIpTableEvent('warn', 'sni_fallback_blocked', { + hostname, + rootDomain, + method, + reason: 'non_idempotent_after_sni_started', }); + throw new OneKeyLocalError( + 'IP Table Adapter: SNI response missing and request is not idempotent', + ); } // Fallback to domain (isFallback = true, so domain failure won't be counted) return await callOriginalAdapter({ @@ -820,6 +1287,15 @@ export function createIpTableAdapter( `[IpTableAdapter] SNI request successful: ${sniResponse.statusCode}`, ); + if (reportRequestSuccessCallback) { + reportRequestSuccessCallback({ + domain: rootDomain, + requestType: 'ip', + target: selectedIp, + requestSequence: sniRequestSequence, + }); + } + // Parse response body let responseData: any = null; const responseBody = sniResponse.body ?? sniResponse.data; @@ -869,14 +1345,27 @@ export function createIpTableAdapter( throw error; } - // Report IP failure if callback is registered - if (reportRequestFailureCallback) { - reportRequestFailureCallback({ - domain: rootDomain, - requestType: 'ip', - target: selectedIp, - error: error instanceof Error ? error.message : String(error), + // Report IP failure if callback is registered (at most once per SNI + // attempt — the null-response branch may already have reported). + reportIpFailureOnce( + error instanceof Error ? error.message : String(error), + ); + + // Once the request was handed to the SNI transport it may have been + // written to the wire (e.g. SNI_TIMEOUT after send). Re-sending over + // the domain risks double-executing mutating requests, so fallback is + // restricted to idempotent methods or errors that prove the + // connection was never established. + const method = (config.method || 'GET').toUpperCase(); + if (!canFallbackAfterSniStarted(method, error)) { + logIpTableEvent('warn', 'sni_fallback_blocked', { + hostname, + rootDomain, + method, + errorCode: getErrorCode(error), + reason: 'non_idempotent_after_sni_started', }); + throw error; } // If SNI request throws error, use original adapter diff --git a/packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts b/packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts new file mode 100644 index 000000000000..108d0e4d7cef --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts @@ -0,0 +1,165 @@ +import { decideEndpoint } from './ipTableEndpointDecision'; + +const base = { + currentSelection: undefined as string | undefined, + domainFailingRealTraffic: false, + strictMode: false, + improvementThreshold: 0.3, +}; + +describe('decideEndpoint', () => { + it('incident regression: domain failing real traffic beats 30% threshold', () => { + // 2026-07-16 incident: domain probe 813ms OK, IPs only ~25% faster, + // but real traffic on the direct domain had failed 23 times in a row. + const d = decideEndpoint({ + ...base, + domainLatency: 813, + ipLatencies: { '216.19.2.116': 610, '104.18.20.233': 640 }, + currentSelection: '', + domainFailingRealTraffic: true, + }); + expect(d).toEqual({ + action: 'select_ip', + ip: '216.19.2.116', + reason: 'domain_failing', + }); + }); + + it('domain probe failed entirely -> best finite ip', () => { + const d = decideEndpoint({ + ...base, + domainLatency: Infinity, + ipLatencies: { 'a': Infinity, 'b': 500 }, + }); + expect(d).toEqual({ + action: 'select_ip', + ip: 'b', + reason: 'domain_probe_failed', + }); + }); + + it('everything failed -> no_change (keep whatever we had)', () => { + const d = decideEndpoint({ + ...base, + domainLatency: Infinity, + ipLatencies: { 'a': Infinity }, + currentSelection: 'a', + }); + expect(d).toEqual({ action: 'no_change', reason: 'all_failed' }); + }); + + it('all ips failed, domain alive -> domain', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 300, + ipLatencies: { 'a': Infinity }, + }); + expect(d).toEqual({ action: 'select_domain', reason: 'all_ip_failed' }); + }); + + it('strict mode always picks best ip when one is finite', () => { + const d = decideEndpoint({ + ...base, + strictMode: true, + domainLatency: 100, + ipLatencies: { 'a': 900 }, + }); + expect(d).toEqual({ action: 'select_ip', ip: 'a', reason: 'strict' }); + }); + + it('fresh choice: ip must be >30% faster to win', () => { + const faster = decideEndpoint({ + ...base, + domainLatency: 1000, + ipLatencies: { 'a': 650 }, + }); + expect(faster).toEqual({ action: 'select_ip', ip: 'a', reason: 'faster' }); + + const competitive = decideEndpoint({ + ...base, + domainLatency: 1000, + ipLatencies: { 'a': 750 }, + }); + expect(competitive).toEqual({ + action: 'select_domain', + reason: 'competitive', + }); + }); + + it('sticky: healthy current ip is kept unless domain is 30% faster', () => { + // domain 800 vs current ip 900 — domain only ~11% faster: keep ip. + const keep = decideEndpoint({ + ...base, + domainLatency: 800, + ipLatencies: { 'cur': 900, 'other': 850 }, + currentSelection: 'cur', + }); + expect(keep).toEqual({ action: 'select_ip', ip: 'cur', reason: 'sticky' }); + + // domain 500 vs ip 900 — domain ~44% faster: switch back to domain. + const back = decideEndpoint({ + ...base, + domainLatency: 500, + ipLatencies: { 'cur': 900 }, + currentSelection: 'cur', + }); + expect(back).toEqual({ action: 'select_domain', reason: 'faster' }); + }); + + it('sticky among ips: another ip must beat current by 30% to displace it', () => { + const keep = decideEndpoint({ + ...base, + domainLatency: 2000, + ipLatencies: { 'cur': 600, 'other': 500 }, + currentSelection: 'cur', + }); + expect(keep).toEqual({ action: 'select_ip', ip: 'cur', reason: 'sticky' }); + + const displace = decideEndpoint({ + ...base, + domainLatency: 2000, + ipLatencies: { 'cur': 600, 'other': 300 }, + currentSelection: 'cur', + }); + expect(displace).toEqual({ + action: 'select_ip', + ip: 'other', + reason: 'faster', + }); + }); + + it('current ip dead -> fresh comparison between best ip and domain', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 400, + ipLatencies: { 'cur': Infinity, 'other': 500 }, + currentSelection: 'cur', + }); + expect(d).toEqual({ action: 'select_domain', reason: 'competitive' }); + }); + + it('explicitly staying on domain reports sticky', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 1000, + ipLatencies: { 'a': 900 }, + currentSelection: '', + }); + expect(d).toEqual({ action: 'select_domain', reason: 'sticky' }); + }); + + it('domain failing overrides sticky-domain even without probe advantage', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 300, + ipLatencies: { 'a': 900 }, + currentSelection: '', + domainFailingRealTraffic: true, + }); + expect(d).toEqual({ + action: 'select_ip', + ip: 'a', + reason: 'domain_failing', + }); + }); +}); diff --git a/packages/shared/src/request/helpers/ipTableEndpointDecision.ts b/packages/shared/src/request/helpers/ipTableEndpointDecision.ts new file mode 100644 index 000000000000..6d04f705cefe --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableEndpointDecision.ts @@ -0,0 +1,125 @@ +import type { + IIpTableEndpointDecision, + IIpTableEndpointDecisionInput, +} from '../types/ipTable'; + +function findBestIp( + ipLatencies: Record, +): { ip: string; latency: number } | null { + let best: { ip: string; latency: number } | null = null; + for (const [ip, latency] of Object.entries(ipLatencies)) { + if (latency !== Infinity && (!best || latency < best.latency)) { + best = { ip, latency }; + } + } + return best; +} + +function isSignificantlyFaster( + candidate: number, + incumbent: number, + threshold: number, +): boolean { + if (incumbent === Infinity) { + return candidate !== Infinity; + } + if (candidate === Infinity) { + return false; + } + return (incumbent - candidate) / incumbent > threshold; +} + +/** + * Pure endpoint selection. Priorities: + * 1. Reachability beats latency: a direct domain that is failing real + * traffic never wins against any reachable IP. + * 2. Hysteresis: the current healthy endpoint keeps its seat unless a + * challenger is `improvementThreshold` faster (both directions). + * 3. The latency threshold only applies to fresh choices between two + * healthy candidates. + */ +export function decideEndpoint( + input: IIpTableEndpointDecisionInput, +): IIpTableEndpointDecision { + const { + domainLatency, + ipLatencies, + currentSelection, + domainFailingRealTraffic, + strictMode, + improvementThreshold, + } = input; + + const bestIp = findBestIp(ipLatencies); + + // Nothing reachable at all — keep the previous selection untouched. + if (domainLatency === Infinity && !bestIp) { + return { action: 'no_change', reason: 'all_failed' }; + } + + // Reachability first: domain probes all failed. + if (domainLatency === Infinity && bestIp) { + return { + action: 'select_ip', + ip: bestIp.ip, + reason: 'domain_probe_failed', + }; + } + + // All IPs failed; domain is the only reachable endpoint. + if (!bestIp) { + return { action: 'select_domain', reason: 'all_ip_failed' }; + } + + if (strictMode) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'strict' }; + } + + // Real traffic on the direct domain is failing: any reachable IP wins, + // regardless of the latency threshold. A health probe succeeding while + // real requests fail is exactly the 2026-07-16 incident pattern. + if (domainFailingRealTraffic) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'domain_failing' }; + } + + const currentIp = + currentSelection && currentSelection !== '' ? currentSelection : null; + const currentIpLatency = currentIp ? ipLatencies[currentIp] : undefined; + const currentIpHealthy = + currentIpLatency !== undefined && currentIpLatency !== Infinity; + + if (currentIp && currentIpHealthy) { + // Sticky-IP: challengers must be significantly faster to displace it. + if ( + isSignificantlyFaster( + domainLatency, + currentIpLatency, + improvementThreshold, + ) + ) { + return { action: 'select_domain', reason: 'faster' }; + } + if ( + bestIp.ip !== currentIp && + isSignificantlyFaster( + bestIp.latency, + currentIpLatency, + improvementThreshold, + ) + ) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'faster' }; + } + return { action: 'select_ip', ip: currentIp, reason: 'sticky' }; + } + + // Fresh choice: on domain, never selected, or the current ip is dead. + if ( + isSignificantlyFaster(bestIp.latency, domainLatency, improvementThreshold) + ) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'faster' }; + } + return { + action: 'select_domain', + reason: currentSelection === '' ? 'sticky' : 'competitive', + }; +} diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts new file mode 100644 index 000000000000..419cc0823637 --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts @@ -0,0 +1,71 @@ +import { + applyOutcome, + createOutcomeLedgerEntry, + nextIpTableRequestSequence, +} from './ipTableOutcomeLedger'; + +describe('ipTableOutcomeLedger', () => { + it('regression: a newer success applied first blocks an older failure arriving late', () => { + // Request A starts at t=1 and fails slowly; request B starts at t=5 and + // succeeds first. B must leave a ledger mark even though the entry was + // fresh, so A's late failure is rejected as stale. + const entry = createOutcomeLedgerEntry(); + expect(applyOutcome(entry, { ok: true, requestSequence: 5 })).toBe( + 'applied', + ); + expect(applyOutcome(entry, { ok: false, requestSequence: 1 })).toBe( + 'stale', + ); + expect(entry.consecutiveFailures).toBe(0); + expect(entry.lastOutcomeSequence).toBe(5); + }); + + it('old failure processed first is overridden by a newer success', () => { + const entry = createOutcomeLedgerEntry(); + expect(applyOutcome(entry, { ok: false, requestSequence: 1 })).toBe( + 'applied', + ); + expect(entry.consecutiveFailures).toBe(1); + expect(applyOutcome(entry, { ok: true, requestSequence: 5 })).toBe( + 'applied', + ); + expect(entry.consecutiveFailures).toBe(0); + }); + + it('both interleavings converge to the same final state', () => { + const a = createOutcomeLedgerEntry(); + applyOutcome(a, { ok: true, requestSequence: 5 }); + applyOutcome(a, { ok: false, requestSequence: 1 }); + + const b = createOutcomeLedgerEntry(); + applyOutcome(b, { ok: false, requestSequence: 1 }); + applyOutcome(b, { ok: true, requestSequence: 5 }); + + expect(a).toEqual(b); + }); + + it('consecutive failures accumulate only for non-stale outcomes', () => { + const entry = createOutcomeLedgerEntry(); + applyOutcome(entry, { ok: false, requestSequence: 10 }); + applyOutcome(entry, { ok: false, requestSequence: 11 }); + // stale failure from before the first one + applyOutcome(entry, { ok: false, requestSequence: 5 }); + expect(entry.consecutiveFailures).toBe(2); + expect(entry.lastOutcomeSequence).toBe(11); + }); + + it('orders requests that start in the same wall-clock millisecond', () => { + jest.spyOn(Date, 'now').mockReturnValue(5); + const earlierSequence = nextIpTableRequestSequence(); + const laterSequence = nextIpTableRequestSequence(); + expect(Date.now()).toBe(5); + expect(laterSequence).toBeGreaterThan(earlierSequence); + + const entry = createOutcomeLedgerEntry(); + applyOutcome(entry, { ok: true, requestSequence: laterSequence }); + expect( + applyOutcome(entry, { ok: false, requestSequence: earlierSequence }), + ).toBe('stale'); + expect(entry.consecutiveFailures).toBe(0); + }); +}); diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts new file mode 100644 index 000000000000..42605c81b6c4 --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts @@ -0,0 +1,46 @@ +/** + * Outcome ledger shared by the adapter's per-hostname fail-open tracking and + * the service's per-endpoint health stats. + * + * Request outcomes arrive through async callbacks with no ordering guarantee: + * an early-started request can fail slowly while a later-started request + * succeeds first. `Date.now()` is not a sufficient ordering key because two + * transports can start in the same millisecond. Every runtime therefore + * assigns a strictly increasing request sequence at transport hand-off; + * ordering by that sequence — never by completion/processing time — makes + * every interleaving converge to the same state. + */ +export interface IIpTableOutcomeLedgerEntry { + /** Consecutive failure count; reset to 0 by an applied success */ + consecutiveFailures: number; + /** Runtime-local request sequence of the newest applied outcome */ + lastOutcomeSequence: number; +} + +export function createOutcomeLedgerEntry(): IIpTableOutcomeLedgerEntry { + return { consecutiveFailures: 0, lastOutcomeSequence: 0 }; +} + +let ipTableRequestSequence = 0; + +/** Allocate at transport hand-off, before awaiting any network work. */ +export function nextIpTableRequestSequence(): number { + ipTableRequestSequence += 1; + return ipTableRequestSequence; +} + +export function applyOutcome( + entry: IIpTableOutcomeLedgerEntry, + outcome: { ok: boolean; requestSequence: number }, +): 'applied' | 'stale' { + if (outcome.requestSequence < entry.lastOutcomeSequence) { + return 'stale'; + } + entry.lastOutcomeSequence = outcome.requestSequence; + if (outcome.ok) { + entry.consecutiveFailures = 0; + } else { + entry.consecutiveFailures += 1; + } + return 'applied'; +} diff --git a/packages/shared/src/request/types/ipTable.ts b/packages/shared/src/request/types/ipTable.ts index f87f491d70ad..0a52fc7c2b96 100644 --- a/packages/shared/src/request/types/ipTable.ts +++ b/packages/shared/src/request/types/ipTable.ts @@ -50,8 +50,74 @@ export interface IIpTableRuntime { selections: { [domain: string]: string; // Currently selected IP for each domain }; + /** + * Best IP measured by the most recent speed test per domain, recorded even + * when the final selection chose the domain. Used for fast failover when + * the direct domain starts failing real traffic. + */ + lastBestIp?: { + [domain: string]: string; + }; + /** Provenance of the last successfully verified CDN config */ + lastVerified?: { + at: number; + version: number; + generatedAt: string; + /** FNV-1a correlation hash of the canonical signed payload (not a security hash) */ + payloadHash: string; + }; +} + +// ==================== Endpoint Decision (pure selection logic) ==================== + +export interface IIpTableEndpointDecisionInput { + /** Average latency of direct-domain probe; Infinity = all probes failed */ + domainLatency: number; + /** ip -> average latency; Infinity = that ip failed all probes */ + ipLatencies: Record; + /** undefined = never selected; '' = domain selected; other = selected ip */ + currentSelection: string | undefined; + /** true when real traffic on direct domain hit the failover threshold */ + domainFailingRealTraffic: boolean; + strictMode: boolean; + /** e.g. 0.3 — candidate must be 30% faster to displace current endpoint */ + improvementThreshold: number; } +export type IIpTableEndpointDecision = + | { + action: 'select_ip'; + ip: string; + reason: + | 'strict' + | 'domain_probe_failed' + | 'domain_failing' + | 'faster' + | 'sticky'; + } + | { + action: 'select_domain'; + reason: 'all_ip_failed' | 'competitive' | 'sticky' | 'faster'; + } + | { action: 'no_change'; reason: 'all_failed' }; + +// ==================== Signature Verification (structured result) ==================== + +export type IIpTableSignatureVerifyFailureReason = + | 'missing_signature' + | 'verifier_load_failed' + | 'malformed_signature' + | 'signer_mismatch' + | 'unexpected_error'; + +export type IIpTableSignatureVerifyResult = + | { ok: true; recoveredAddress: string } + | { + ok: false; + reason: IIpTableSignatureVerifyFailureReason; + errorMessage?: string; + }; + /** * IP Table configuration with runtime state * Clear separation between config and runtime data diff --git a/packages/shared/src/utils/ipTableUtils.test.ts b/packages/shared/src/utils/ipTableUtils.test.ts index 16aca7fcfaf7..4313044e45ef 100644 --- a/packages/shared/src/utils/ipTableUtils.test.ts +++ b/packages/shared/src/utils/ipTableUtils.test.ts @@ -1,8 +1,16 @@ -import { DEFAULT_IP_TABLE_CONFIG } from '../request/constants/ipTableDefaults'; +import { OneKeyLocalError } from '../errors'; +import { + CDN_SIGNER_ADDRESS, + DEFAULT_IP_TABLE_CONFIG, +} from '../request/constants/ipTableDefaults'; import { - mergeIpTableConfigs, + computeIpTableConfigHash, + isIpTableConfigRegression, + isValidIpTableRemoteConfigShape, + pruneIpTableRuntimeSelections, verifyIpTableConfigSignature, + verifyIpTableConfigSignatureDetailed, } from './ipTableUtils'; import type { IIpTableRemoteConfig } from '../request/types/ipTable'; @@ -140,353 +148,297 @@ describe('verifyIpTableConfigSignature', () => { }); }); -describe('mergeIpTableConfigs', () => { - test('merges new domains from remote config', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'local.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'remote.com': { - endpoints: [ - { ip: '2.2.2.2', provider: 'remote', region: 'CN', weight: 50 }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['local.com']).toBeDefined(); - expect(merged.domains['remote.com']).toBeDefined(); - expect(merged.domains['local.com'].endpoints).toHaveLength(1); - expect(merged.domains['remote.com'].endpoints).toHaveLength(1); +describe('verifyIpTableConfigSignatureDetailed', () => { + test('builtin config verifies and recovers the CDN signer', async () => { + const result = await verifyIpTableConfigSignatureDetailed( + DEFAULT_IP_TABLE_CONFIG, + ); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.recoveredAddress.toLowerCase()).toBe( + CDN_SIGNER_ADDRESS.toLowerCase(), + ); + } }); - test('merges new endpoints for existing domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'example.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'example.com': { - endpoints: [ - { ip: '2.2.2.2', provider: 'remote', region: 'CN', weight: 50 }, - ], - }, - }, - }; + test('extra unknown top-level fields do not break verification (allowlist canonicalization)', async () => { + // The 2026-07 CDN pipeline may attach metadata fields the signer never + // covered. Verification must canonicalize only the signed fields. + const configWithExtras = { + ...DEFAULT_IP_TABLE_CONFIG, + updated_by: 'cdn-pipeline', + etag: 'abc123', + } as unknown as IIpTableRemoteConfig; - const merged = mergeIpTableConfigs(localConfig, remoteConfig); + const result = await verifyIpTableConfigSignatureDetailed(configWithExtras); + expect(result.ok).toBe(true); + }); - expect(merged.domains['example.com'].endpoints).toHaveLength(2); - expect(merged.domains['example.com'].endpoints[0].ip).toBe('1.1.1.1'); - expect(merged.domains['example.com'].endpoints[1].ip).toBe('2.2.2.2'); + test('missing signature -> reason missing_signature', async () => { + const config = { + ...DEFAULT_IP_TABLE_CONFIG, + signature: '', + } as IIpTableRemoteConfig; + const result = await verifyIpTableConfigSignatureDetailed(config); + expect(result).toMatchObject({ ok: false, reason: 'missing_signature' }); }); - test('deduplicates endpoints with same IP', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', + test('tampered domains -> reason signer_mismatch', async () => { + const tampered: IIpTableRemoteConfig = { + ...DEFAULT_IP_TABLE_CONFIG, domains: { - 'example.com': { + ...DEFAULT_IP_TABLE_CONFIG.domains, + 'evil.com': { endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - { ip: '2.2.2.2', provider: 'local2', region: 'GLOBAL', weight: 80 }, + { ip: '6.6.6.6', provider: 'evil', region: 'GLOBAL', weight: 100 }, ], }, }, }; + const result = await verifyIpTableConfigSignatureDetailed(tampered); + expect(result).toMatchObject({ ok: false, reason: 'signer_mismatch' }); + }); - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'example.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'remote', region: 'CN', weight: 50 }, // Duplicate IP - { ip: '3.3.3.3', provider: 'remote2', region: 'CN', weight: 60 }, // New IP - ], - }, - }, + test('garbage signature -> reason malformed_signature', async () => { + const config: IIpTableRemoteConfig = { + ...DEFAULT_IP_TABLE_CONFIG, + signature: 'not-a-valid-hex-string', }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - // Should have 3 endpoints: 1.1.1.1 (from local), 2.2.2.2 (from local), 3.3.3.3 (from remote) - expect(merged.domains['example.com'].endpoints).toHaveLength(3); - - const ips = merged.domains['example.com'].endpoints.map((ep) => ep.ip); - expect(ips).toEqual(['1.1.1.1', '2.2.2.2', '3.3.3.3']); + const result = await verifyIpTableConfigSignatureDetailed(config); + expect(result).toMatchObject({ ok: false, reason: 'malformed_signature' }); }); - test('uses remote config metadata (version, ttl, generated_at, signature)', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'local.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; + test('verifier import failure -> reason verifier_load_failed (split-bundle incident)', async () => { + // Reproduces the 2026-07-16 incident: the background runtime rejected + // loading the @ethersproject segment; verification must surface a + // packaging problem instead of pretending the signature is invalid. + await jest.isolateModulesAsync(async () => { + jest.doMock('@ethersproject/wallet', () => { + throw new OneKeyLocalError( + "Segment runtime 'main' not allowed in 'background' runtime", + ); + }); + // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require + const isolated = + require('./ipTableUtils') as typeof import('./ipTableUtils'); + const result = await isolated.verifyIpTableConfigSignatureDetailed( + DEFAULT_IP_TABLE_CONFIG, + ); + expect(result).toMatchObject({ + ok: false, + reason: 'verifier_load_failed', + }); + jest.dontMock('@ethersproject/wallet'); + }); + }); +}); - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'remote.com': { - endpoints: [ - { ip: '2.2.2.2', provider: 'remote', region: 'CN', weight: 50 }, - ], - }, - }, - }; +describe('isValidIpTableRemoteConfigShape', () => { + test('accepts the builtin config', () => { + expect(isValidIpTableRemoteConfigShape(DEFAULT_IP_TABLE_CONFIG)).toBe(true); + }); - const merged = mergeIpTableConfigs(localConfig, remoteConfig); + test('rejects primitives and null', () => { + expect(isValidIpTableRemoteConfigShape(null)).toBe(false); + expect(isValidIpTableRemoteConfigShape(undefined)).toBe(false); + expect(isValidIpTableRemoteConfigShape('oops')).toBe(false); + expect(isValidIpTableRemoteConfigShape(42)).toBe(false); + }); - expect(merged.version).toBe(2); - expect(merged.ttl_sec).toBe(43_200); - expect(merged.generated_at).toBe('2025-11-06T12:00:00.000Z'); - expect(merged.signature).toBe('0xremote'); + test('rejects wrong field types', () => { + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + version: '1', + }), + ).toBe(false); + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + ttl_sec: '86400', + }), + ).toBe(false); + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + domains: null, + }), + ).toBe(false); }); - test('preserves all local endpoints when merging', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'example.com': { - endpoints: [ - { - ip: '1.1.1.1', - provider: 'provider1', - region: 'GLOBAL', - weight: 100, - }, - { - ip: '2.2.2.2', - provider: 'provider2', - region: 'GLOBAL', - weight: 90, - }, - { ip: '3.3.3.3', provider: 'provider3', region: 'CN', weight: 80 }, - ], - }, - }, - }; + test('rejects malformed endpoints', () => { + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + domains: { 'example.com': { endpoints: 'not-an-array' } }, + }), + ).toBe(false); + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + domains: { 'example.com': { endpoints: [{ ip: 123 }] } }, + }), + ).toBe(false); + }); +}); - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'example.com': { - endpoints: [ - { - ip: '4.4.4.4', - provider: 'provider4', - region: 'GLOBAL', - weight: 70, - }, - ], - }, - }, +describe('isIpTableConfigRegression', () => { + function makeConfig(version: number, generatedAt: string) { + return { + ...DEFAULT_IP_TABLE_CONFIG, + version, + generated_at: generatedAt, }; + } + + test('without lastVerified, the active config generated_at anchors same-version comparison', () => { + // Pre-upgrade installs have no runtime.lastVerified; a same-version but + // older signed config must still be rejected against the active config. + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2025-10-01T00:00:00.000Z'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + lastVerified: undefined, + }); + expect(result).toEqual({ + regression: true, + reason: 'generated_at_regression', + }); + }); - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['example.com'].endpoints).toHaveLength(4); - - // Verify all local endpoints are preserved - const ips = merged.domains['example.com'].endpoints.map((ep) => ep.ip); - expect(ips).toContain('1.1.1.1'); - expect(ips).toContain('2.2.2.2'); - expect(ips).toContain('3.3.3.3'); - expect(ips).toContain('4.4.4.4'); + test('same version with newer generated_at is accepted', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2026-01-01T00:00:00.000Z'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + }); + expect(result).toEqual({ regression: false }); }); - test('handles empty local domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: {}, - }; + test('lower version is always a regression', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2026-01-01T00:00:00.000Z'), + localConfig: makeConfig(2, '2025-11-06T08:30:54.066Z'), + }); + expect(result).toEqual({ regression: true, reason: 'version_regression' }); + }); - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'remote.com': { - endpoints: [ - { - ip: '1.1.1.1', - provider: 'remote', - region: 'GLOBAL', - weight: 100, - }, - ], - }, - }, - }; + test('unparseable remote generated_at is rejected', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(2, 'not-a-date'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + }); + expect(result).toEqual({ + regression: true, + reason: 'unparseable_generated_at', + }); + }); - const merged = mergeIpTableConfigs(localConfig, remoteConfig); + test('lastVerified anchor takes precedence over the active config', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2025-12-01T00:00:00.000Z'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + lastVerified: { version: 1, generatedAt: '2026-01-01T00:00:00.000Z' }, + }); + expect(result).toEqual({ + regression: true, + reason: 'generated_at_regression', + }); + }); - expect(merged.domains['remote.com']).toBeDefined(); - expect(merged.domains['remote.com'].endpoints).toHaveLength(1); + test('higher version wins regardless of generated_at', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(3, '2025-01-01T00:00:00.000Z'), + localConfig: makeConfig(2, '2025-11-06T08:30:54.066Z'), + lastVerified: { version: 2, generatedAt: '2025-11-06T08:30:54.066Z' }, + }); + expect(result).toEqual({ regression: false }); }); +}); - test('handles empty remote domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'local.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; +describe('computeIpTableConfigHash', () => { + test('is stable and ignores unsigned extra fields', () => { + const base = computeIpTableConfigHash(DEFAULT_IP_TABLE_CONFIG); + const withExtras = computeIpTableConfigHash({ + ...DEFAULT_IP_TABLE_CONFIG, + updated_by: 'cdn-pipeline', + } as unknown as IIpTableRemoteConfig); + expect(base).toMatch(/^[0-9a-f]{8}$/); + expect(withExtras).toBe(base); + }); - const remoteConfig: IIpTableRemoteConfig = { + test('changes when signed content changes', () => { + const base = computeIpTableConfigHash(DEFAULT_IP_TABLE_CONFIG); + const changed = computeIpTableConfigHash({ + ...DEFAULT_IP_TABLE_CONFIG, version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: {}, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['local.com']).toBeDefined(); - expect(merged.domains['local.com'].endpoints).toHaveLength(1); + }); + expect(changed).not.toBe(base); }); +}); - test('handles complex merge scenario with multiple domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'onekeycn.com': { - endpoints: [ - { - ip: '104.18.20.233', - provider: 'cloudflare', - region: 'GLOBAL', - weight: 100, - }, - { - ip: '216.19.4.106', - provider: 'volcengine', - region: 'CN', - weight: 100, - }, - ], - }, - 'localonly.com': { - endpoints: [ - { ip: '5.5.5.5', provider: 'local', region: 'GLOBAL', weight: 50 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'onekeycn.com': { - endpoints: [ - { - ip: '104.18.20.233', - provider: 'cloudflare', - region: 'GLOBAL', - weight: 100, - }, // Duplicate - { ip: '6.6.6.6', provider: 'newcdn', region: 'CN', weight: 90 }, // New - ], - }, - 'remoteonly.com': { - endpoints: [ - { ip: '7.7.7.7', provider: 'remote', region: 'GLOBAL', weight: 80 }, - ], - }, +describe('pruneIpTableRuntimeSelections', () => { + const config: IIpTableRemoteConfig = { + version: 2, + ttl_sec: 86_400, + generated_at: '2026-07-17T00:00:00.000Z', + signature: '0xnew', + domains: { + 'onekeycn.com': { + endpoints: [ + { ip: '1.1.1.1', provider: 'a', region: 'GLOBAL', weight: 100 }, + { ip: '2.2.2.2', provider: 'b', region: 'CN', weight: 100 }, + ], }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(Object.keys(merged.domains)).toHaveLength(3); + }, + }; + + test('keeps selections and last-best ips the config still endorses', () => { + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'onekeycn.com': '1.1.1.1' }, + lastBestIp: { 'onekeycn.com': '2.2.2.2' }, + }); + expect(result.selections).toEqual({ 'onekeycn.com': '1.1.1.1' }); + expect(result.lastBestIp).toEqual({ 'onekeycn.com': '2.2.2.2' }); + expect(result.prunedCount).toBe(0); + }); - expect(merged.domains['onekeycn.com'].endpoints).toHaveLength(3); - const onekeyIps = merged.domains['onekeycn.com'].endpoints.map( - (ep) => ep.ip, - ); - expect(onekeyIps).toEqual(['104.18.20.233', '216.19.4.106', '6.6.6.6']); + test('drops selections pointing at revoked endpoints', () => { + // 9.9.9.9 was removed from the signed config (revoked / rotated out): + // neither the selection nor the last-best ip may keep routing to it. + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'onekeycn.com': '9.9.9.9' }, + lastBestIp: { 'onekeycn.com': '9.9.9.9' }, + }); + expect(result.selections).toEqual({}); + expect(result.lastBestIp).toEqual({}); + expect(result.prunedCount).toBe(2); + }); - expect(merged.domains['localonly.com'].endpoints).toHaveLength(1); - expect(merged.domains['localonly.com'].endpoints[0].ip).toBe('5.5.5.5'); + test('always keeps the explicit domain choice (empty string)', () => { + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'onekeycn.com': '' }, + }); + expect(result.selections).toEqual({ 'onekeycn.com': '' }); + expect(result.prunedCount).toBe(0); + }); - expect(merged.domains['remoteonly.com'].endpoints).toHaveLength(1); - expect(merged.domains['remoteonly.com'].endpoints[0].ip).toBe('7.7.7.7'); + test('drops state for domains the config no longer covers', () => { + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'gone.com': '1.1.1.1' }, + lastBestIp: { 'gone.com': '1.1.1.1' }, + }); + expect(result.selections).toEqual({}); + expect(result.lastBestIp).toEqual({}); + expect(result.prunedCount).toBe(2); + }); - // Metadata should be from remote - expect(merged.version).toBe(2); - expect(merged.ttl_sec).toBe(43_200); - expect(merged.signature).toBe('0xremote'); + test('handles missing runtime maps', () => { + const result = pruneIpTableRuntimeSelections({ config }); + expect(result.selections).toEqual({}); + expect(result.lastBestIp).toEqual({}); + expect(result.prunedCount).toBe(0); }); }); diff --git a/packages/shared/src/utils/ipTableUtils.ts b/packages/shared/src/utils/ipTableUtils.ts index 5bf21063a7ed..b15255cf9f7f 100644 --- a/packages/shared/src/utils/ipTableUtils.ts +++ b/packages/shared/src/utils/ipTableUtils.ts @@ -2,85 +2,271 @@ // (and its heavy transitive deps) into the common startup bundle. import stringify from 'fast-json-stable-stringify'; +import { defaultLogger } from '../logger/logger'; import platformEnv from '../platformEnv'; import { CDN_SIGNER_ADDRESS } from '../request/constants/ipTableDefaults'; -import type { IIpTableRemoteConfig } from '../request/types/ipTable'; +import type { + IIpTableRemoteConfig, + IIpTableSignatureVerifyResult, +} from '../request/types/ipTable'; export function isSupportIpTablePlatform() { return platformEnv.isNative || platformEnv.isDesktop; } -export async function verifyIpTableConfigSignature( - config: IIpTableRemoteConfig, -): Promise { - try { - const { signature, ...dataToVerify } = config; +/** + * Fields covered by the CDN signature. Extra metadata fields the pipeline may + * add later must NOT break verification, so we canonicalize an explicit + * allowlist instead of spreading "everything except signature". + */ +function buildCanonicalSignedPayload(config: IIpTableRemoteConfig): string { + return stringify({ + version: config.version, + ttl_sec: config.ttl_sec, + generated_at: config.generated_at, + domains: config.domains, + }); +} + +/** FNV-1a 32-bit — correlation id for logs/persistence, not a security hash. */ +export function computeIpTableConfigHash(config: IIpTableRemoteConfig): string { + const value = buildCanonicalSignedPayload(config); + let hash = 0x81_1c_9d_c5; + for (let i = 0; i < value.length; i += 1) { + hash ^= value.charCodeAt(i); + hash = Math.imul(hash, 0x01_00_01_93) >>> 0; + } + return hash.toString(16).padStart(8, '0'); +} - if (!signature) { - console.error( - '[IpTableUtils] Signature verification failed: Missing signature', - ); +/** + * Structural guard for CDN payloads. A misconfigured edge can return HTML or + * a truncated body that still parses; reject it before verification so the + * failure reason points at the delivery layer, not the signature. + */ +export function isValidIpTableRemoteConfigShape( + data: unknown, +): data is IIpTableRemoteConfig { + if (!data || typeof data !== 'object') { + return false; + } + const config = data as Partial; + if (typeof config.version !== 'number') { + return false; + } + if (typeof config.ttl_sec !== 'number') { + return false; + } + if (typeof config.generated_at !== 'string') { + return false; + } + if (typeof config.signature !== 'string') { + return false; + } + if (!config.domains || typeof config.domains !== 'object') { + return false; + } + for (const domainConfig of Object.values(config.domains)) { + if (!domainConfig || typeof domainConfig !== 'object') { return false; } + const endpoints = (domainConfig as { endpoints?: unknown }).endpoints; + if (!Array.isArray(endpoints)) { + return false; + } + for (const endpoint of endpoints) { + if (!endpoint || typeof endpoint !== 'object') { + return false; + } + if (typeof (endpoint as { ip?: unknown }).ip !== 'string') { + return false; + } + } + } + return true; +} - const canonicalString = stringify(dataToVerify); +function logVerifyEvent( + level: 'info' | 'error', + fields: Record, +): void { + const info = `[IpTableUtils] ${Object.entries({ + event: 'iptable_config_verify', + ...fields, + }) + .map(([key, value]) => `${key}=${String(value)}`) + .join(' ')}`; + if (level === 'error') { + defaultLogger.ipTable.request.error({ info }); + } else { + defaultLogger.ipTable.request.info({ info }); + } +} - const { verifyMessage } = await import('@ethersproject/wallet'); - const recoveredAddress = verifyMessage(canonicalString, signature); +/** + * Verify the CDN config signature with a structured result that separates + * the failure stages. `verifier_load_failed` means the crypto verifier could + * not even be loaded in this runtime (e.g. a split-bundle segment refused for + * the background runtime — the 2026-07-16 incident); it is a packaging + * problem, NOT an invalid signature, and must never be reported as one. + */ +export async function verifyIpTableConfigSignatureDetailed( + config: IIpTableRemoteConfig, +): Promise { + const payloadHash = computeIpTableConfigHash(config); - const isValid = - recoveredAddress.toLowerCase() === CDN_SIGNER_ADDRESS.toLowerCase(); + if (!config.signature) { + logVerifyEvent('error', { + stage: 'precheck', + reason: 'missing_signature', + configVersion: config.version, + payloadHash, + }); + return { ok: false, reason: 'missing_signature' }; + } - if (!isValid) { - console.error( - '[IpTableUtils] Signature verification failed: Invalid signer', - '\n Expected:', - CDN_SIGNER_ADDRESS, - '\n Recovered:', - recoveredAddress, - ); - } + const canonicalString = buildCanonicalSignedPayload(config); - return isValid; + let verifyMessage: (message: string, signature: string) => string; + try { + ({ verifyMessage } = await import('@ethersproject/wallet')); } catch (error) { - console.error('[IpTableUtils] Signature verification error:', error); - return false; + const errorMessage = error instanceof Error ? error.message : String(error); + logVerifyEvent('error', { + stage: 'load_verifier', + reason: 'verifier_load_failed', + configVersion: config.version, + payloadHash, + errorMessage, + }); + return { ok: false, reason: 'verifier_load_failed', errorMessage }; } + + try { + const recoveredAddress = verifyMessage(canonicalString, config.signature); + if (recoveredAddress.toLowerCase() === CDN_SIGNER_ADDRESS.toLowerCase()) { + logVerifyEvent('info', { + stage: 'compare_signer', + result: 'ok', + configVersion: config.version, + generatedAt: config.generated_at, + payloadHash, + }); + return { ok: true, recoveredAddress }; + } + // Signer addresses are public constants; logging them is safe and is the + // single most useful datum when diagnosing a bad CDN publish. + logVerifyEvent('error', { + stage: 'compare_signer', + reason: 'signer_mismatch', + configVersion: config.version, + payloadHash, + recovered: recoveredAddress, + expected: CDN_SIGNER_ADDRESS, + }); + return { ok: false, reason: 'signer_mismatch' }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logVerifyEvent('error', { + stage: 'recover', + reason: 'malformed_signature', + configVersion: config.version, + payloadHash, + errorMessage, + }); + return { ok: false, reason: 'malformed_signature', errorMessage }; + } +} + +export async function verifyIpTableConfigSignature( + config: IIpTableRemoteConfig, +): Promise { + const result = await verifyIpTableConfigSignatureDetailed(config); + return result.ok; } -export function mergeIpTableConfigs( - localConfig: IIpTableRemoteConfig, - remoteConfig: IIpTableRemoteConfig, -): IIpTableRemoteConfig { - const mergedDomains = { ...localConfig.domains }; +/** + * Rollback protection: a valid signature is not enough — a stale but + * legitimately signed config must not displace a newer one. When no + * `lastVerified` anchor exists yet (installs that predate provenance + * tracking), the currently active config's own version/generated_at serves + * as the anchor, so the monotonic guarantee holds from the first update. + */ +export function isIpTableConfigRegression(options: { + remoteConfig: IIpTableRemoteConfig; + localConfig: IIpTableRemoteConfig; + lastVerified?: { version: number; generatedAt: string }; +}): { regression: boolean; reason?: string } { + const { remoteConfig, localConfig, lastVerified } = options; - for (const [domain, remoteDomainConfig] of Object.entries( - remoteConfig.domains, - )) { - if (mergedDomains[domain]) { - const localEndpoints = mergedDomains[domain].endpoints; - const remoteEndpoints = remoteDomainConfig.endpoints; + const remoteGeneratedAtMs = Date.parse(remoteConfig.generated_at); + if (Number.isNaN(remoteGeneratedAtMs)) { + return { regression: true, reason: 'unparseable_generated_at' }; + } - const existingIps = new Set(localEndpoints.map((ep) => ep.ip)); + if (remoteConfig.version < localConfig.version) { + return { regression: true, reason: 'version_regression' }; + } - const newEndpoints = remoteEndpoints.filter( - (ep) => !existingIps.has(ep.ip), - ); + const anchorVersion = lastVerified?.version ?? localConfig.version; + const anchorGeneratedAtMs = Date.parse( + lastVerified?.generatedAt ?? localConfig.generated_at, + ); + if ( + remoteConfig.version === anchorVersion && + !Number.isNaN(anchorGeneratedAtMs) && + remoteGeneratedAtMs < anchorGeneratedAtMs + ) { + return { regression: true, reason: 'generated_at_regression' }; + } + + return { regression: false }; +} + +/** + * Drop runtime selections and last-best IPs that the current signed config + * no longer endorses. The persisted effective config is the signed remote + * envelope verbatim (never an additive merge), so a revoked or compromised + * endpoint removed from the CDN config must also stop being routable via + * previously persisted runtime state. + */ +export function pruneIpTableRuntimeSelections(options: { + config: IIpTableRemoteConfig; + selections?: Record; + lastBestIp?: Record; +}): { + selections: Record; + lastBestIp: Record; + prunedCount: number; +} { + const allowedIpsByDomain = new Map>(); + for (const [domain, domainConfig] of Object.entries(options.config.domains)) { + allowedIpsByDomain.set( + domain, + new Set(domainConfig.endpoints.map((endpoint) => endpoint.ip)), + ); + } + + let prunedCount = 0; + const selections: Record = {}; + for (const [domain, ip] of Object.entries(options.selections ?? {})) { + // '' is the explicit "use the domain" choice and is always allowed. + if (ip === '' || allowedIpsByDomain.get(domain)?.has(ip)) { + selections[domain] = ip; + } else { + prunedCount += 1; + } + } - mergedDomains[domain] = { - endpoints: [...localEndpoints, ...newEndpoints], - }; + const lastBestIp: Record = {}; + for (const [domain, ip] of Object.entries(options.lastBestIp ?? {})) { + if (allowedIpsByDomain.get(domain)?.has(ip)) { + lastBestIp[domain] = ip; } else { - mergedDomains[domain] = remoteDomainConfig; + prunedCount += 1; } } - return { - version: remoteConfig.version, - ttl_sec: remoteConfig.ttl_sec, - generated_at: remoteConfig.generated_at, - signature: remoteConfig.signature, - domains: mergedDomains, - }; + return { selections, lastBestIp, prunedCount }; }