From f1a6b7cbdaa609fe271ba62974906def27654dc1 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 19:46:33 +0800 Subject: [PATCH 01/22] feat: add ip table endpoint decision and verify result types --- .../src/request/constants/ipTableDefaults.ts | 17 +++++ packages/shared/src/request/types/ipTable.ts | 66 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/packages/shared/src/request/constants/ipTableDefaults.ts b/packages/shared/src/request/constants/ipTableDefaults.ts index 727815e36d80..a13b64298ff1 100644 --- a/packages/shared/src/request/constants/ipTableDefaults.ts +++ b/packages/shared/src/request/constants/ipTableDefaults.ts @@ -16,6 +16,23 @@ export const IP_TABLE_SPEED_TEST_COOLDOWN_MS = timerUtils.getTimeDurationMs({ minute: 2, }); +/** + * Domain Failover Threshold + * After this many consecutive real-request network failures on the direct + * domain, immediately switch selection to the last known-good IP (service) + * or fail-open to a builtin IP (adapter) without waiting for a speed test. + */ +export const IP_TABLE_DOMAIN_FAILOVER_THRESHOLD = 3; + +/** + * Adapter fail-open window. While active, hosts of the affected root domain + * resolve to a fallback IP even when no runtime selection exists (covers the + * main runtime and the cold-start window where simpleDb config is absent). + */ +export const IP_TABLE_ADAPTER_FAILOVER_TTL_MS = timerUtils.getTimeDurationMs({ + minute: 5, +}); + /** * Performance Improvement Threshold * Only use IP routing if it's at least this percentage faster than direct domain access diff --git a/packages/shared/src/request/types/ipTable.ts b/packages/shared/src/request/types/ipTable.ts index f87f491d70ad..0a52fc7c2b96 100644 --- a/packages/shared/src/request/types/ipTable.ts +++ b/packages/shared/src/request/types/ipTable.ts @@ -50,8 +50,74 @@ export interface IIpTableRuntime { selections: { [domain: string]: string; // Currently selected IP for each domain }; + /** + * Best IP measured by the most recent speed test per domain, recorded even + * when the final selection chose the domain. Used for fast failover when + * the direct domain starts failing real traffic. + */ + lastBestIp?: { + [domain: string]: string; + }; + /** Provenance of the last successfully verified CDN config */ + lastVerified?: { + at: number; + version: number; + generatedAt: string; + /** FNV-1a correlation hash of the canonical signed payload (not a security hash) */ + payloadHash: string; + }; +} + +// ==================== Endpoint Decision (pure selection logic) ==================== + +export interface IIpTableEndpointDecisionInput { + /** Average latency of direct-domain probe; Infinity = all probes failed */ + domainLatency: number; + /** ip -> average latency; Infinity = that ip failed all probes */ + ipLatencies: Record; + /** undefined = never selected; '' = domain selected; other = selected ip */ + currentSelection: string | undefined; + /** true when real traffic on direct domain hit the failover threshold */ + domainFailingRealTraffic: boolean; + strictMode: boolean; + /** e.g. 0.3 — candidate must be 30% faster to displace current endpoint */ + improvementThreshold: number; } +export type IIpTableEndpointDecision = + | { + action: 'select_ip'; + ip: string; + reason: + | 'strict' + | 'domain_probe_failed' + | 'domain_failing' + | 'faster' + | 'sticky'; + } + | { + action: 'select_domain'; + reason: 'all_ip_failed' | 'competitive' | 'sticky' | 'faster'; + } + | { action: 'no_change'; reason: 'all_failed' }; + +// ==================== Signature Verification (structured result) ==================== + +export type IIpTableSignatureVerifyFailureReason = + | 'missing_signature' + | 'verifier_load_failed' + | 'malformed_signature' + | 'signer_mismatch' + | 'unexpected_error'; + +export type IIpTableSignatureVerifyResult = + | { ok: true; recoveredAddress: string } + | { + ok: false; + reason: IIpTableSignatureVerifyFailureReason; + errorMessage?: string; + }; + /** * IP Table configuration with runtime state * Clear separation between config and runtime data From 39af8130841785ee729e3bf27ad1f5f3e12030c3 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 19:46:44 +0800 Subject: [PATCH 02/22] feat: add pure ip table endpoint decision with reachability-first and hysteresis --- .../helpers/ipTableEndpointDecision.test.ts | 165 ++++++++++++++++++ .../helpers/ipTableEndpointDecision.ts | 125 +++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts create mode 100644 packages/shared/src/request/helpers/ipTableEndpointDecision.ts diff --git a/packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts b/packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts new file mode 100644 index 000000000000..108d0e4d7cef --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableEndpointDecision.test.ts @@ -0,0 +1,165 @@ +import { decideEndpoint } from './ipTableEndpointDecision'; + +const base = { + currentSelection: undefined as string | undefined, + domainFailingRealTraffic: false, + strictMode: false, + improvementThreshold: 0.3, +}; + +describe('decideEndpoint', () => { + it('incident regression: domain failing real traffic beats 30% threshold', () => { + // 2026-07-16 incident: domain probe 813ms OK, IPs only ~25% faster, + // but real traffic on the direct domain had failed 23 times in a row. + const d = decideEndpoint({ + ...base, + domainLatency: 813, + ipLatencies: { '216.19.2.116': 610, '104.18.20.233': 640 }, + currentSelection: '', + domainFailingRealTraffic: true, + }); + expect(d).toEqual({ + action: 'select_ip', + ip: '216.19.2.116', + reason: 'domain_failing', + }); + }); + + it('domain probe failed entirely -> best finite ip', () => { + const d = decideEndpoint({ + ...base, + domainLatency: Infinity, + ipLatencies: { 'a': Infinity, 'b': 500 }, + }); + expect(d).toEqual({ + action: 'select_ip', + ip: 'b', + reason: 'domain_probe_failed', + }); + }); + + it('everything failed -> no_change (keep whatever we had)', () => { + const d = decideEndpoint({ + ...base, + domainLatency: Infinity, + ipLatencies: { 'a': Infinity }, + currentSelection: 'a', + }); + expect(d).toEqual({ action: 'no_change', reason: 'all_failed' }); + }); + + it('all ips failed, domain alive -> domain', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 300, + ipLatencies: { 'a': Infinity }, + }); + expect(d).toEqual({ action: 'select_domain', reason: 'all_ip_failed' }); + }); + + it('strict mode always picks best ip when one is finite', () => { + const d = decideEndpoint({ + ...base, + strictMode: true, + domainLatency: 100, + ipLatencies: { 'a': 900 }, + }); + expect(d).toEqual({ action: 'select_ip', ip: 'a', reason: 'strict' }); + }); + + it('fresh choice: ip must be >30% faster to win', () => { + const faster = decideEndpoint({ + ...base, + domainLatency: 1000, + ipLatencies: { 'a': 650 }, + }); + expect(faster).toEqual({ action: 'select_ip', ip: 'a', reason: 'faster' }); + + const competitive = decideEndpoint({ + ...base, + domainLatency: 1000, + ipLatencies: { 'a': 750 }, + }); + expect(competitive).toEqual({ + action: 'select_domain', + reason: 'competitive', + }); + }); + + it('sticky: healthy current ip is kept unless domain is 30% faster', () => { + // domain 800 vs current ip 900 — domain only ~11% faster: keep ip. + const keep = decideEndpoint({ + ...base, + domainLatency: 800, + ipLatencies: { 'cur': 900, 'other': 850 }, + currentSelection: 'cur', + }); + expect(keep).toEqual({ action: 'select_ip', ip: 'cur', reason: 'sticky' }); + + // domain 500 vs ip 900 — domain ~44% faster: switch back to domain. + const back = decideEndpoint({ + ...base, + domainLatency: 500, + ipLatencies: { 'cur': 900 }, + currentSelection: 'cur', + }); + expect(back).toEqual({ action: 'select_domain', reason: 'faster' }); + }); + + it('sticky among ips: another ip must beat current by 30% to displace it', () => { + const keep = decideEndpoint({ + ...base, + domainLatency: 2000, + ipLatencies: { 'cur': 600, 'other': 500 }, + currentSelection: 'cur', + }); + expect(keep).toEqual({ action: 'select_ip', ip: 'cur', reason: 'sticky' }); + + const displace = decideEndpoint({ + ...base, + domainLatency: 2000, + ipLatencies: { 'cur': 600, 'other': 300 }, + currentSelection: 'cur', + }); + expect(displace).toEqual({ + action: 'select_ip', + ip: 'other', + reason: 'faster', + }); + }); + + it('current ip dead -> fresh comparison between best ip and domain', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 400, + ipLatencies: { 'cur': Infinity, 'other': 500 }, + currentSelection: 'cur', + }); + expect(d).toEqual({ action: 'select_domain', reason: 'competitive' }); + }); + + it('explicitly staying on domain reports sticky', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 1000, + ipLatencies: { 'a': 900 }, + currentSelection: '', + }); + expect(d).toEqual({ action: 'select_domain', reason: 'sticky' }); + }); + + it('domain failing overrides sticky-domain even without probe advantage', () => { + const d = decideEndpoint({ + ...base, + domainLatency: 300, + ipLatencies: { 'a': 900 }, + currentSelection: '', + domainFailingRealTraffic: true, + }); + expect(d).toEqual({ + action: 'select_ip', + ip: 'a', + reason: 'domain_failing', + }); + }); +}); diff --git a/packages/shared/src/request/helpers/ipTableEndpointDecision.ts b/packages/shared/src/request/helpers/ipTableEndpointDecision.ts new file mode 100644 index 000000000000..6d04f705cefe --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableEndpointDecision.ts @@ -0,0 +1,125 @@ +import type { + IIpTableEndpointDecision, + IIpTableEndpointDecisionInput, +} from '../types/ipTable'; + +function findBestIp( + ipLatencies: Record, +): { ip: string; latency: number } | null { + let best: { ip: string; latency: number } | null = null; + for (const [ip, latency] of Object.entries(ipLatencies)) { + if (latency !== Infinity && (!best || latency < best.latency)) { + best = { ip, latency }; + } + } + return best; +} + +function isSignificantlyFaster( + candidate: number, + incumbent: number, + threshold: number, +): boolean { + if (incumbent === Infinity) { + return candidate !== Infinity; + } + if (candidate === Infinity) { + return false; + } + return (incumbent - candidate) / incumbent > threshold; +} + +/** + * Pure endpoint selection. Priorities: + * 1. Reachability beats latency: a direct domain that is failing real + * traffic never wins against any reachable IP. + * 2. Hysteresis: the current healthy endpoint keeps its seat unless a + * challenger is `improvementThreshold` faster (both directions). + * 3. The latency threshold only applies to fresh choices between two + * healthy candidates. + */ +export function decideEndpoint( + input: IIpTableEndpointDecisionInput, +): IIpTableEndpointDecision { + const { + domainLatency, + ipLatencies, + currentSelection, + domainFailingRealTraffic, + strictMode, + improvementThreshold, + } = input; + + const bestIp = findBestIp(ipLatencies); + + // Nothing reachable at all — keep the previous selection untouched. + if (domainLatency === Infinity && !bestIp) { + return { action: 'no_change', reason: 'all_failed' }; + } + + // Reachability first: domain probes all failed. + if (domainLatency === Infinity && bestIp) { + return { + action: 'select_ip', + ip: bestIp.ip, + reason: 'domain_probe_failed', + }; + } + + // All IPs failed; domain is the only reachable endpoint. + if (!bestIp) { + return { action: 'select_domain', reason: 'all_ip_failed' }; + } + + if (strictMode) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'strict' }; + } + + // Real traffic on the direct domain is failing: any reachable IP wins, + // regardless of the latency threshold. A health probe succeeding while + // real requests fail is exactly the 2026-07-16 incident pattern. + if (domainFailingRealTraffic) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'domain_failing' }; + } + + const currentIp = + currentSelection && currentSelection !== '' ? currentSelection : null; + const currentIpLatency = currentIp ? ipLatencies[currentIp] : undefined; + const currentIpHealthy = + currentIpLatency !== undefined && currentIpLatency !== Infinity; + + if (currentIp && currentIpHealthy) { + // Sticky-IP: challengers must be significantly faster to displace it. + if ( + isSignificantlyFaster( + domainLatency, + currentIpLatency, + improvementThreshold, + ) + ) { + return { action: 'select_domain', reason: 'faster' }; + } + if ( + bestIp.ip !== currentIp && + isSignificantlyFaster( + bestIp.latency, + currentIpLatency, + improvementThreshold, + ) + ) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'faster' }; + } + return { action: 'select_ip', ip: currentIp, reason: 'sticky' }; + } + + // Fresh choice: on domain, never selected, or the current ip is dead. + if ( + isSignificantlyFaster(bestIp.latency, domainLatency, improvementThreshold) + ) { + return { action: 'select_ip', ip: bestIp.ip, reason: 'faster' }; + } + return { + action: 'select_domain', + reason: currentSelection === '' ? 'sticky' : 'competitive', + }; +} From 19aaf4b855517687968034712d5f97b5ce6fced9 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 19:50:31 +0800 Subject: [PATCH 03/22] feat: reachability-first endpoint selection with fast domain failover --- .../simple/entity/SimpleDbEntityIpTable.ts | 30 ++++ .../kit-bg/src/services/ServiceIpTable.ts | 167 +++++++++--------- .../src/states/jotai/atoms/devSettings.ts | 4 + 3 files changed, 122 insertions(+), 79 deletions(-) diff --git a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts index b0f5a545dbb3..5277a4500fdf 100644 --- a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts +++ b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts @@ -105,6 +105,36 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase { + await this.setRawData((data) => { + const runtime = data?.runtime ?? { + enabled: true, + lastUpdated: 0, + lastRegionCheck: 0, + selections: {}, + }; + + return { + ...data, + config: data?.config ?? null, + currentRegion: data?.currentRegion ?? 'AUTO', + runtime: { + ...runtime, + lastBestIp: { + ...runtime.lastBestIp, + [domain]: ip, + }, + }, + version: data?.version ?? 1, + }; + }); + } + @backgroundMethod() async setEnabled(enabled: boolean): Promise { await this.setRawData((data) => ({ diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 567db9578029..8366d32cba7f 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -14,6 +14,7 @@ import { } from '@onekeyhq/shared/src/config/appConfig'; import { defaultLogger } from '@onekeyhq/shared/src/logger/logger'; import { + IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, IP_TABLE_INITIAL_SPEED_TEST_DELAY_MS, IP_TABLE_PERFORMANCE_IMPROVEMENT_THRESHOLD, IP_TABLE_SNI_FAILURE_THRESHOLD, @@ -28,6 +29,7 @@ import { testDomainSpeed, testIpSpeed, } from '@onekeyhq/shared/src/request/helpers/ipTableAdapter'; +import { decideEndpoint } from '@onekeyhq/shared/src/request/helpers/ipTableEndpointDecision'; import { isSniSupported } from '@onekeyhq/shared/src/request/helpers/sniRequest'; import { getRequestHeaders } from '@onekeyhq/shared/src/request/Interceptor'; import type { @@ -182,14 +184,12 @@ class ServiceIpTable extends ServiceBase { const configWithRuntime = await this.getConfig(); const currentSelection = configWithRuntime.runtime?.selections?.[domain]; - // Check if current active endpoint is failing - if (currentSelection === undefined) { - // No selection yet, don't trigger - return false; - } - - if (currentSelection === '') { - // Using domain directly + // Check if current active endpoint is failing. + // undefined means no selection was ever made — real traffic goes direct + // to the domain in that state, so judge by the domain's health instead of + // never triggering (the old `return false` deadlocked selection forever + // when the initial speed test had not completed). + if (currentSelection === undefined || currentSelection === '') { return ( stats.domainDirect.consecutiveFailures >= IP_TABLE_SNI_FAILURE_THRESHOLD ); @@ -486,11 +486,16 @@ class ServiceIpTable extends ServiceBase { /** * Select best endpoint for a domain - * Compares domain direct connection vs all IPs with SNI - * Prefers domain if IP is not significantly faster (30% threshold) + * Compares domain direct connection vs all IPs with SNI via the pure + * decision policy (reachability-first + hysteresis). When triggered by + * real-traffic domain failures, any reachable IP wins regardless of the + * latency threshold. */ @backgroundMethod() - async selectBestEndpointForDomain(domain: string): Promise { + async selectBestEndpointForDomain( + domain: string, + opts?: { trigger?: 'domain_failure' | 'ip_failure' | 'periodic' }, + ): Promise { // Check if IP Table is enabled if (!(await this.isIpTableEnabled())) { defaultLogger.ipTable.request.info({ @@ -554,88 +559,56 @@ class ServiceIpTable extends ServiceBase { }); } - // 3. Find best IP - let bestIp = ''; - let bestIpLatency = Infinity; + // 3. Decide via pure policy (reachability-first + hysteresis) + const { enabled: devSettingEnabled, settings } = + await devSettingsPersistAtom.get(); + const strictMode = Boolean( + devSettingEnabled && settings?.forceIpTableStrict, + ); + const currentSelection = configWithRuntime.runtime?.selections?.[domain]; + const ipLatencies: Record = {}; for (const [ip, latency] of ipResults) { - if (latency < bestIpLatency) { - bestIpLatency = latency; - bestIp = ip; - } - } - - // 4. Compare and decide - if (domainLatency === Infinity) { - // Domain test failed - if (bestIpLatency !== Infinity) { - // Use best IP - defaultLogger.ipTable.request.info({ - info: `[IpTable] Domain failed, using IP: ${domain} -> ${bestIp}`, - }); - await this.backgroundApi.simpleDb.ipTable.updateSelection( - domain, - bestIp, - ); - } else { - // All tests failed - defaultLogger.ipTable.request.info({ - info: `[IpTable] All tests failed for ${domain}`, - }); - } - return; + ipLatencies[ip] = latency; } - if (bestIpLatency === Infinity) { - // All IP tests failed, use domain - defaultLogger.ipTable.request.info({ - info: `[IpTable] All IP tests failed, using domain: ${domain}`, - }); - await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ''); - return; - } + const decision = decideEndpoint({ + domainLatency, + ipLatencies, + currentSelection, + domainFailingRealTraffic: opts?.trigger === 'domain_failure', + strictMode, + improvementThreshold: IP_TABLE_PERFORMANCE_IMPROVEMENT_THRESHOLD, + }); - // Check if forceIpTableStrict mode is enabled - const { enabled: devSettingEnabled, settings } = - await devSettingsPersistAtom.get(); - const forceIpTableStrict = - devSettingEnabled && settings?.forceIpTableStrict; - // In strict mode, always use IP if available (regardless of domain speed) - if (forceIpTableStrict) { - defaultLogger.ipTable.request.info({ - info: `[IpTable] [STRICT MODE] Using best IP: ${domain} -> ${bestIp} (${bestIpLatency}ms)`, - }); - await this.backgroundApi.simpleDb.ipTable.updateSelection( + // Persist the best measured IP for fast failover, regardless of the + // final decision — it is the candidate we switch to when the direct + // domain starts failing real traffic. + const bestMeasuredIp = Object.entries(ipLatencies) + .filter(([, latency]) => latency !== Infinity) + .toSorted((a, b) => a[1] - b[1])[0]?.[0]; + if (bestMeasuredIp) { + await this.backgroundApi.simpleDb.ipTable.updateLastBestIp( domain, - bestIp, + bestMeasuredIp, ); - return; } - // Normal mode: use threshold-based decision - // Calculate performance improvement - const improvement = (domainLatency - bestIpLatency) / domainLatency; + defaultLogger.ipTable.request.info({ + info: `[IpTable] Endpoint decision for ${domain}: ${decision.action} (${ + decision.reason + }) trigger=${opts?.trigger ?? 'periodic'} domainLatency=${domainLatency}ms`, + }); - if (improvement > IP_TABLE_PERFORMANCE_IMPROVEMENT_THRESHOLD) { - // IP is significantly faster (>30%), use IP - defaultLogger.ipTable.request.info({ - info: `[IpTable] IP is ${(improvement * 100).toFixed( - 1, - )}% faster, using IP: ${domain} -> ${bestIp}`, - }); + if (decision.action === 'select_ip') { await this.backgroundApi.simpleDb.ipTable.updateSelection( domain, - bestIp, + decision.ip, ); - } else { - // Domain is competitive, prefer domain for stability - defaultLogger.ipTable.request.info({ - info: `[IpTable] Domain is competitive (IP only ${( - improvement * 100 - ).toFixed(1)}% faster), using domain: ${domain}`, - }); + } else if (decision.action === 'select_domain') { await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ''); } + // no_change: keep the previous selection untouched. // Clean up health stats for unused endpoints after speed test await this.cleanupHealthStatsAfterSpeedTest(domain); @@ -718,6 +691,28 @@ class ServiceIpTable extends ServiceBase { }) for ${domain} (${target})`, }); + // Fast failover: after a few consecutive real-request failures on the + // direct domain, switch to the last known-good IP immediately instead of + // waiting for the slow failure threshold + a full speed test round. + if ( + requestType === 'domain' && + endpointHealth.consecutiveFailures === IP_TABLE_DOMAIN_FAILOVER_THRESHOLD + ) { + const configWithRuntime = await this.getConfig(); + const currentSelection = configWithRuntime.runtime?.selections?.[domain]; + const lastBestIp = configWithRuntime.runtime?.lastBestIp?.[domain]; + const failoverDisabled = await this.isFailoverDisabledByDevSettings(); + if (!failoverDisabled && (currentSelection ?? '') === '' && lastBestIp) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Fast failover: domain ${domain} failing real traffic, switching to last-best IP immediately`, + }); + await this.backgroundApi.simpleDb.ipTable.updateSelection( + domain, + lastBestIp, + ); + } + } + // Check if speed test should be triggered const shouldTrigger = await this.shouldTriggerSpeedTest(domain, stats); @@ -753,7 +748,21 @@ class ServiceIpTable extends ServiceBase { }); // Trigger speed test to find and switch to better endpoint - void this.selectBestEndpointForDomain(domain); + void this.selectBestEndpointForDomain(domain, { + trigger: requestType === 'domain' ? 'domain_failure' : 'ip_failure', + }); + } + + private async isFailoverDisabledByDevSettings(): Promise { + try { + const devSettings = await devSettingsPersistAtom.get(); + return Boolean( + devSettings.enabled && devSettings.settings?.disableIpTableFailover, + ); + } catch { + // Default to enabled when dev settings are unreadable. + return false; + } } private scheduleSpeedTest(reason: string): void { diff --git a/packages/kit-bg/src/states/jotai/atoms/devSettings.ts b/packages/kit-bg/src/states/jotai/atoms/devSettings.ts index 01b132c2d257..0793a2fb2728 100644 --- a/packages/kit-bg/src/states/jotai/atoms/devSettings.ts +++ b/packages/kit-bg/src/states/jotai/atoms/devSettings.ts @@ -93,6 +93,9 @@ export interface IDevSettings { // Force IP Table strict mode: always use IP even if runtime.selections is empty // Fallback to first available IP from config when no selection exists forceIpTableStrict?: boolean; + // Kill switch for the fast-failover behaviors introduced for extreme + // network conditions (adapter fail-open + service fast switch to last-best IP) + disableIpTableFailover?: boolean; // Enable mock market banner data for UI testing enableMockMarketBanner?: boolean; // Test accounts for OneKey ID login testing @@ -172,6 +175,7 @@ export const { usbCommunicationMode: 'webusb', disableIpTableInProd: false, // IP Table enabled by default forceIpTableStrict: false, // Strict mode: disabled by default + disableIpTableFailover: false, // Fast failover enabled by default useFastPbkdf2NativeBackend: false, }, }, From 95a229a16afed264b802c6eddedd14e484b90cc8 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 19:51:42 +0800 Subject: [PATCH 04/22] feat: coalesce concurrent ip table speed tests per domain --- .../kit-bg/src/services/ServiceIpTable.ts | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 8366d32cba7f..92ef0e15a4dc 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -484,6 +484,11 @@ class ServiceIpTable extends ServiceBase { } } + // Coalesce concurrent speed tests per domain (anti probe-storm): the log + // of the 2026-07-16 incident shows ~20 concurrent rounds saturating an + // already congested network and polluting the latency measurements. + private speedTestInFlight = new Map>(); + /** * Select best endpoint for a domain * Compares domain direct connection vs all IPs with SNI via the pure @@ -495,6 +500,26 @@ class ServiceIpTable extends ServiceBase { async selectBestEndpointForDomain( domain: string, opts?: { trigger?: 'domain_failure' | 'ip_failure' | 'periodic' }, + ): Promise { + const inflight = this.speedTestInFlight.get(domain); + if (inflight) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Speed test already in flight for ${domain}, coalescing`, + }); + return inflight; + } + const run = this.selectBestEndpointForDomainInternal(domain, opts).finally( + () => { + this.speedTestInFlight.delete(domain); + }, + ); + this.speedTestInFlight.set(domain, run); + return run; + } + + private async selectBestEndpointForDomainInternal( + domain: string, + opts?: { trigger?: 'domain_failure' | 'ip_failure' | 'periodic' }, ): Promise { // Check if IP Table is enabled if (!(await this.isIpTableEnabled())) { @@ -765,13 +790,23 @@ class ServiceIpTable extends ServiceBase { } } + private pendingInitialSpeedTestTimer: ReturnType | null = + null; + private scheduleSpeedTest(reason: string): void { + if (this.pendingInitialSpeedTestTimer) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] ${reason}, speed test already scheduled, skipping`, + }); + return; + } defaultLogger.ipTable.request.info({ info: `[IpTable] ${reason}, scheduling speed test in ${ IP_TABLE_INITIAL_SPEED_TEST_DELAY_MS / 1000 } s`, }); - setTimeout(() => { + this.pendingInitialSpeedTestTimer = setTimeout(() => { + this.pendingInitialSpeedTestTimer = null; void this.runFullSpeedTest(); }, IP_TABLE_INITIAL_SPEED_TEST_DELAY_MS); } From f163b494b67b192a19799ad647e79140bb562e90 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 19:56:50 +0800 Subject: [PATCH 05/22] feat: adapter-level fail-open to builtin ip on domain network failures --- .../request/helpers/ipTableAdapter.test.ts | 228 ++++++++++++++++- .../src/request/helpers/ipTableAdapter.ts | 238 +++++++++++++++++- 2 files changed, 458 insertions(+), 8 deletions(-) diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index 4ddeb1b8a74a..024a35edd978 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -3,7 +3,11 @@ import axios from 'axios'; import { getRequestHeaders } from '../Interceptor'; import requestHelper from '../requestHelper'; -import { createIpTableAdapter, testIpSpeed } from './ipTableAdapter'; +import { + createIpTableAdapter, + resetAdapterFailoverStatesForTesting, + testIpSpeed, +} from './ipTableAdapter'; import { isProxyActiveForUrl, isSniSupported, sniRequest } from './sniRequest'; import type { AxiosResponse, InternalAxiosRequestConfig } from 'axios'; @@ -228,3 +232,225 @@ describe('ipTableAdapter SNI preflight and fail-closed behavior', () => { ); }); }); + +describe('ipTableAdapter fail-open on domain network failures', () => { + const BUILTIN_CN_IPS = [ + '104.18.20.233', + '104.18.21.233', + '216.19.3.115', + '216.19.2.116', + '216.19.4.106', + ]; + + let originalAdapter: typeof axios.defaults.adapter; + let fallbackAdapter: jest.Mock< + Promise, + [InternalAxiosRequestConfig] + >; + let consoleErrorSpy: jest.SpyInstance; + + function networkError() { + // Transport-level failure: no HTTP response attached. + return Object.assign(new Error('timeout of 30000ms exceeded'), { + code: 'ECONNABORTED', + }); + } + + function httpError(status: number) { + return Object.assign(new Error(`Request failed with status ${status}`), { + response: { status }, + }); + } + + beforeEach(() => { + consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + originalAdapter = axios.defaults.adapter; + fallbackAdapter = jest.fn(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + axios.defaults.adapter = fallbackAdapter; + + mockedIsSniSupported.mockReturnValue(true); + mockedIsProxyActiveForUrl.mockResolvedValue(false); + mockedSniRequest.mockReset(); + mockedGetRequestHeaders.mockResolvedValue({}); + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + settings: {}, + } as never); + // Simulate the main runtime / cold-start window: no config installed. + mockedRequestHelper.getIpTableConfig.mockResolvedValue(null as never); + resetAdapterFailoverStatesForTesting(); + }); + + afterEach(() => { + axios.defaults.adapter = originalAdapter; + consoleErrorSpy.mockRestore(); + resetAdapterFailoverStatesForTesting(); + jest.restoreAllMocks(); + jest.clearAllMocks(); + }); + + async function failNTimes(n: number) { + fallbackAdapter.mockImplementation(async () => { + throw networkError(); + }); + for (let i = 0; i < n; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + } + + test('activates fail-open after 3 consecutive network errors and routes next request via builtin ip', async () => { + await failNTimes(3); + expect(mockedSniRequest).not.toHaveBeenCalled(); + + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ status: 200, data: { ok: true } }); + + expect(mockedSniRequest).toHaveBeenCalledTimes(1); + const sniArgs = mockedSniRequest.mock.calls[0][0] as { + ip: string; + hostname: string; + }; + expect(BUILTIN_CN_IPS).toContain(sniArgs.ip); + expect(sniArgs.hostname).toBe('wallet.onekeycn.com'); + }); + + test('does not replay the failing request itself', async () => { + await failNTimes(3); + // Every one of the 3 failing requests must reject; no hidden retry via SNI. + expect(fallbackAdapter).toHaveBeenCalledTimes(3); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('http error responses do not count as network failures', async () => { + fallbackAdapter.mockImplementation(async () => { + throw httpError(500); + }); + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ response: { status: 500 } }); + } + + // Next request still goes direct domain: fail-open must not be active. + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('a success resets the consecutive failure counter', async () => { + await failNTimes(2); + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ); + await failNTimes(2); + + // 2 + reset + 2 consecutive failures: threshold (3) not reached. + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('fail-open expires after its TTL window', async () => { + await failNTimes(3); + + const realNow = Date.now(); + const nowSpy = jest + .spyOn(Date, 'now') + .mockReturnValue(realNow + 5 * 60_000 + 1); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + + nowSpy.mockRestore(); + }); + + test('kill switch disables fail-open entirely', async () => { + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + enabled: true, + settings: { disableIpTableFailover: true }, + } as never); + + await failNTimes(3); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 5075ff71f07b..79dc3e5a8c8f 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -4,6 +4,11 @@ import { OneKeyLocalError } from '../../errors'; import { defaultLogger } from '../../logger/logger'; import platformEnv from '../../platformEnv'; import { memoizee } from '../../utils/cacheUtils'; +import { + DEFAULT_IP_TABLE_CONFIG, + IP_TABLE_ADAPTER_FAILOVER_TTL_MS, + IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, +} from '../constants/ipTableDefaults'; import { getRequestHeaders } from '../Interceptor'; import requestHelper from '../requestHelper'; @@ -149,6 +154,157 @@ async function getMappedDomainForIpLookup( } } +// getSelectedIpForHostInternal is a hoisted function declaration, so wiring +// the memoized wrapper up here (before the fail-open helpers that clear its +// cache) is safe at module-evaluation time. +const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { + promise: true, + maxAge: 5000, // 5 seconds cache + max: 100, // Max 100 hostname cached + primitive: true, // hostname is a string primitive, use simple equality check +}); + +// ========== Fail-open on domain network failures ========== +// +// When real requests on the direct domain keep failing at the transport +// level (timeouts, resets — NOT HTTP error responses), the adapter opens a +// time-boxed window during which hosts of that root domain resolve to a +// fallback IP (runtime last-best IP when available, builtin config +// otherwise). This is the only protection available in runtimes where the +// simpleDb-backed config is absent: the main runtime on split-runtime +// targets and the cold-start window before background init completes. +// Failing requests are never replayed — a wallet must not double-send +// mutating POSTs — only subsequent requests take the fail-open route. + +interface IAdapterFailoverState { + consecutiveNetworkFailures: number; + failOpenUntil: number; + /** Rotates across activations so a dead builtin endpoint is not retried forever */ + fallbackIpIndex: number; +} + +const adapterFailoverStates = new Map(); + +/** Test-only helper: clears fail-open state and the selection memo cache. */ +export function resetAdapterFailoverStatesForTesting(): void { + adapterFailoverStates.clear(); + getSelectedIpForHost.clear(); +} + +function isNetworkLevelError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + // An HTTP response means the network path works; only transport-level + // failures (timeout, reset, dns) should trigger fail-open. + if ('response' in error && (error as { response?: unknown }).response) { + return false; + } + return true; +} + +async function isFailoverDisabledByDevSettings(): Promise { + try { + const devSettings = await requestHelper.getDevSettingsPersistAtom(); + return !!devSettings.settings?.disableIpTableFailover; + } catch { + // Default to enabled when dev settings are unreadable. + return false; + } +} + +function getFailoverState(lookupDomain: string): IAdapterFailoverState { + let state = adapterFailoverStates.get(lookupDomain); + if (!state) { + state = { + consecutiveNetworkFailures: 0, + failOpenUntil: 0, + fallbackIpIndex: 0, + }; + adapterFailoverStates.set(lookupDomain, state); + } + return state; +} + +function isFailoverActive(lookupDomain: string): boolean { + const state = adapterFailoverStates.get(lookupDomain); + return Boolean(state && state.failOpenUntil > Date.now()); +} + +async function resolveFailoverLookupDomain( + rootDomain: string, +): Promise { + const mappedDomain = await getMappedDomainForIpLookup(rootDomain); + return mappedDomain || rootDomain; +} + +async function recordDomainRequestOutcome(options: { + rootDomain: string; + ok: boolean; + error?: unknown; +}): Promise { + const { rootDomain, ok, error } = options; + const lookupDomain = await resolveFailoverLookupDomain(rootDomain); + const state = getFailoverState(lookupDomain); + + if (ok) { + if (state.consecutiveNetworkFailures > 0 || state.failOpenUntil > 0) { + if (state.failOpenUntil > Date.now()) { + logIpTableEvent('info', 'adapter_failover_deactivated', { + lookupDomain, + cause: 'domain_recovered', + }); + } + state.consecutiveNetworkFailures = 0; + state.failOpenUntil = 0; + getSelectedIpForHost.clear(); + } + return; + } + + if (!isNetworkLevelError(error)) { + return; + } + if (await isFailoverDisabledByDevSettings()) { + return; + } + + state.consecutiveNetworkFailures += 1; + if ( + state.consecutiveNetworkFailures >= IP_TABLE_DOMAIN_FAILOVER_THRESHOLD && + state.failOpenUntil <= Date.now() + ) { + state.failOpenUntil = Date.now() + IP_TABLE_ADAPTER_FAILOVER_TTL_MS; + state.fallbackIpIndex += 1; + getSelectedIpForHost.clear(); + logIpTableEvent('warn', 'adapter_failover_activated', { + lookupDomain, + consecutiveNetworkFailures: state.consecutiveNetworkFailures, + ttlMs: IP_TABLE_ADAPTER_FAILOVER_TTL_MS, + }); + } +} + +function getFailoverFallbackIp(lookupDomain: string): string | null { + const endpoints = DEFAULT_IP_TABLE_CONFIG.domains[lookupDomain]?.endpoints; + if (!endpoints || endpoints.length === 0) { + return null; + } + const state = getFailoverState(lookupDomain); + return endpoints[state.fallbackIpIndex % endpoints.length].ip; +} + +/** + * Resolve the fail-open IP for a lookup domain, preferring the last-best IP + * measured by the background speed test when a runtime is available. + */ +function resolveFailoverIp( + lookupDomain: string, + runtimeLastBestIp: string | undefined, +): string | null { + return runtimeLastBestIp || getFailoverFallbackIp(lookupDomain); +} + /** * Check if IP Table should be used based on environment and dev settings * @returns true if IP Table should be used, false otherwise @@ -216,6 +372,29 @@ async function getSelectedIpForHostInternal( // Check if config exists and is enabled if (!configWithRuntime || configWithRuntime.runtime?.enabled === false) { + // No config = the main runtime or the cold-start window before the + // background init completes. When the domain is failing at transport + // level, fail-open to a builtin IP so those runtimes have protection + // too. runtime.enabled === false is explicit user intent: no fail-open. + if (!configWithRuntime) { + const lookupDomain = await resolveFailoverLookupDomain(rootDomain); + if (isFailoverActive(lookupDomain)) { + const failoverIp = resolveFailoverIp(lookupDomain, undefined); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: lookupDomain !== rootDomain, + strictMode: false, + runtimeEnabled: false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; + } + } + } logIpTableEvent('info', 'iptable_selection', { hostname, rootDomain, @@ -269,6 +448,25 @@ async function getSelectedIpForHostInternal( // In strict mode, override this and use fallback IP from config if (selectedIp === '') { if (!strictMode) { + if (isFailoverActive(lookupDomain)) { + const failoverIp = resolveFailoverIp( + lookupDomain, + runtime?.lastBestIp?.[lookupDomain], + ); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: Boolean(mappedDomain), + strictMode: false, + runtimeEnabled: runtime?.enabled !== false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; + } + } debugLog( `[IpTableAdapter] Explicitly using domain for: ${lookupDomain}`, ); @@ -308,6 +506,26 @@ async function getSelectedIpForHostInternal( } } + if (isFailoverActive(lookupDomain)) { + const failoverIp = resolveFailoverIp( + lookupDomain, + runtime?.lastBestIp?.[lookupDomain], + ); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: Boolean(mappedDomain), + strictMode: Boolean(strictMode), + runtimeEnabled: runtime?.enabled !== false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; + } + } + logIpTableEvent('info', 'iptable_selection', { hostname, rootDomain, @@ -344,13 +562,6 @@ async function getSelectedIpForHostInternal( } } -const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { - promise: true, - maxAge: 5000, // 5 seconds cache - max: 100, // Max 100 hostname cached - primitive: true, // hostname is a string primitive, use simple equality check -}); - /** * Convert AxiosHeaders to plain object */ @@ -521,8 +732,21 @@ export function createIpTableAdapter( ); } + if (rootDomain) { + // A completed domain request (any HTTP status) proves the network + // path works: reset the adapter fail-open counter. + await recordDomainRequestOutcome({ rootDomain, ok: true }); + } + return response; } catch (error) { + if (rootDomain) { + // Track transport-level failures for adapter fail-open. The failing + // request itself is never replayed (no double-send of POSTs); only + // subsequent requests take the fail-open route. + await recordDomainRequestOutcome({ rootDomain, ok: false, error }); + } + // Only report domain failures if this is NOT a fallback request if ( !isFallback && From 285b3c724baa2ee18b4dd1c132585662b2b5aff3 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 20:03:34 +0800 Subject: [PATCH 06/22] feat: structured ip table config verification with allowlist canonicalization --- .../kit-bg/src/services/ServiceIpTable.ts | 23 ++- .../shared/src/utils/ipTableUtils.test.ts | 164 ++++++++++++++- packages/shared/src/utils/ipTableUtils.ts | 191 +++++++++++++++--- 3 files changed, 346 insertions(+), 32 deletions(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 92ef0e15a4dc..73e64f302beb 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -38,8 +38,9 @@ import type { } from '@onekeyhq/shared/src/request/types/ipTable'; import { isSupportIpTablePlatform, + isValidIpTableRemoteConfigShape, mergeIpTableConfigs, - verifyIpTableConfigSignature, + verifyIpTableConfigSignatureDetailed, } from '@onekeyhq/shared/src/utils/ipTableUtils'; import { devSettingsPersistAtom } from '../states/jotai/atoms'; @@ -316,6 +317,16 @@ class ServiceIpTable extends ServiceBase { return null; } + // A misconfigured CDN edge can return HTML or a truncated body that + // still parses; reject it here so the log points at the delivery + // layer instead of a bogus "signature verification failed". + if (!isValidIpTableRemoteConfigShape(remoteConfig)) { + defaultLogger.ipTable.request.error({ + info: '[IpTable] Skipping CDN config update: invalid config shape (delivery-layer problem)', + }); + return null; + } + defaultLogger.ipTable.request.info({ info: `[IpTable] Remote config fetched successfully, version: ${remoteConfig.version}`, }); @@ -356,11 +367,15 @@ class ServiceIpTable extends ServiceBase { return false; } - const isValidSignature = await verifyIpTableConfigSignature(remoteConfig); + const verifyResult = + await verifyIpTableConfigSignatureDetailed(remoteConfig); - if (!isValidSignature) { + if (!verifyResult.ok) { + // Keep the failure stage in the message: `verifier_load_failed` is a + // packaging/runtime problem (e.g. split-bundle segment refused), + // NOT an untrusted signature — they need different owners. defaultLogger.ipTable.request.error({ - info: '[IpTable] Skipping CDN config update: signature verification failed', + info: `[IpTable] Skipping CDN config update: signature verification failed, reason=${verifyResult.reason}`, }); return false; } diff --git a/packages/shared/src/utils/ipTableUtils.test.ts b/packages/shared/src/utils/ipTableUtils.test.ts index 16aca7fcfaf7..6b7b0b66c37a 100644 --- a/packages/shared/src/utils/ipTableUtils.test.ts +++ b/packages/shared/src/utils/ipTableUtils.test.ts @@ -1,8 +1,15 @@ -import { DEFAULT_IP_TABLE_CONFIG } from '../request/constants/ipTableDefaults'; +import { OneKeyLocalError } from '../errors'; +import { + CDN_SIGNER_ADDRESS, + DEFAULT_IP_TABLE_CONFIG, +} from '../request/constants/ipTableDefaults'; import { + computeIpTableConfigHash, + isValidIpTableRemoteConfigShape, mergeIpTableConfigs, verifyIpTableConfigSignature, + verifyIpTableConfigSignatureDetailed, } from './ipTableUtils'; import type { IIpTableRemoteConfig } from '../request/types/ipTable'; @@ -140,6 +147,161 @@ describe('verifyIpTableConfigSignature', () => { }); }); +describe('verifyIpTableConfigSignatureDetailed', () => { + test('builtin config verifies and recovers the CDN signer', async () => { + const result = await verifyIpTableConfigSignatureDetailed( + DEFAULT_IP_TABLE_CONFIG, + ); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.recoveredAddress.toLowerCase()).toBe( + CDN_SIGNER_ADDRESS.toLowerCase(), + ); + } + }); + + test('extra unknown top-level fields do not break verification (allowlist canonicalization)', async () => { + // The 2026-07 CDN pipeline may attach metadata fields the signer never + // covered. Verification must canonicalize only the signed fields. + const configWithExtras = { + ...DEFAULT_IP_TABLE_CONFIG, + updated_by: 'cdn-pipeline', + etag: 'abc123', + } as unknown as IIpTableRemoteConfig; + + const result = await verifyIpTableConfigSignatureDetailed(configWithExtras); + expect(result.ok).toBe(true); + }); + + test('missing signature -> reason missing_signature', async () => { + const config = { + ...DEFAULT_IP_TABLE_CONFIG, + signature: '', + } as IIpTableRemoteConfig; + const result = await verifyIpTableConfigSignatureDetailed(config); + expect(result).toMatchObject({ ok: false, reason: 'missing_signature' }); + }); + + test('tampered domains -> reason signer_mismatch', async () => { + const tampered: IIpTableRemoteConfig = { + ...DEFAULT_IP_TABLE_CONFIG, + domains: { + ...DEFAULT_IP_TABLE_CONFIG.domains, + 'evil.com': { + endpoints: [ + { ip: '6.6.6.6', provider: 'evil', region: 'GLOBAL', weight: 100 }, + ], + }, + }, + }; + const result = await verifyIpTableConfigSignatureDetailed(tampered); + expect(result).toMatchObject({ ok: false, reason: 'signer_mismatch' }); + }); + + test('garbage signature -> reason malformed_signature', async () => { + const config: IIpTableRemoteConfig = { + ...DEFAULT_IP_TABLE_CONFIG, + signature: 'not-a-valid-hex-string', + }; + const result = await verifyIpTableConfigSignatureDetailed(config); + expect(result).toMatchObject({ ok: false, reason: 'malformed_signature' }); + }); + + test('verifier import failure -> reason verifier_load_failed (split-bundle incident)', async () => { + // Reproduces the 2026-07-16 incident: the background runtime rejected + // loading the @ethersproject segment; verification must surface a + // packaging problem instead of pretending the signature is invalid. + await jest.isolateModulesAsync(async () => { + jest.doMock('@ethersproject/wallet', () => { + throw new OneKeyLocalError( + "Segment runtime 'main' not allowed in 'background' runtime", + ); + }); + // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require + const isolated = + require('./ipTableUtils') as typeof import('./ipTableUtils'); + const result = await isolated.verifyIpTableConfigSignatureDetailed( + DEFAULT_IP_TABLE_CONFIG, + ); + expect(result).toMatchObject({ + ok: false, + reason: 'verifier_load_failed', + }); + jest.dontMock('@ethersproject/wallet'); + }); + }); +}); + +describe('isValidIpTableRemoteConfigShape', () => { + test('accepts the builtin config', () => { + expect(isValidIpTableRemoteConfigShape(DEFAULT_IP_TABLE_CONFIG)).toBe(true); + }); + + test('rejects primitives and null', () => { + expect(isValidIpTableRemoteConfigShape(null)).toBe(false); + expect(isValidIpTableRemoteConfigShape(undefined)).toBe(false); + expect(isValidIpTableRemoteConfigShape('oops')).toBe(false); + expect(isValidIpTableRemoteConfigShape(42)).toBe(false); + }); + + test('rejects wrong field types', () => { + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + version: '1', + }), + ).toBe(false); + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + ttl_sec: '86400', + }), + ).toBe(false); + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + domains: null, + }), + ).toBe(false); + }); + + test('rejects malformed endpoints', () => { + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + domains: { 'example.com': { endpoints: 'not-an-array' } }, + }), + ).toBe(false); + expect( + isValidIpTableRemoteConfigShape({ + ...DEFAULT_IP_TABLE_CONFIG, + domains: { 'example.com': { endpoints: [{ ip: 123 }] } }, + }), + ).toBe(false); + }); +}); + +describe('computeIpTableConfigHash', () => { + test('is stable and ignores unsigned extra fields', () => { + const base = computeIpTableConfigHash(DEFAULT_IP_TABLE_CONFIG); + const withExtras = computeIpTableConfigHash({ + ...DEFAULT_IP_TABLE_CONFIG, + updated_by: 'cdn-pipeline', + } as unknown as IIpTableRemoteConfig); + expect(base).toMatch(/^[0-9a-f]{8}$/); + expect(withExtras).toBe(base); + }); + + test('changes when signed content changes', () => { + const base = computeIpTableConfigHash(DEFAULT_IP_TABLE_CONFIG); + const changed = computeIpTableConfigHash({ + ...DEFAULT_IP_TABLE_CONFIG, + version: 2, + }); + expect(changed).not.toBe(base); + }); +}); + describe('mergeIpTableConfigs', () => { test('merges new domains from remote config', () => { const localConfig: IIpTableRemoteConfig = { diff --git a/packages/shared/src/utils/ipTableUtils.ts b/packages/shared/src/utils/ipTableUtils.ts index 5bf21063a7ed..819c3bbdace7 100644 --- a/packages/shared/src/utils/ipTableUtils.ts +++ b/packages/shared/src/utils/ipTableUtils.ts @@ -2,53 +2,190 @@ // (and its heavy transitive deps) into the common startup bundle. import stringify from 'fast-json-stable-stringify'; +import { defaultLogger } from '../logger/logger'; import platformEnv from '../platformEnv'; import { CDN_SIGNER_ADDRESS } from '../request/constants/ipTableDefaults'; -import type { IIpTableRemoteConfig } from '../request/types/ipTable'; +import type { + IIpTableRemoteConfig, + IIpTableSignatureVerifyResult, +} from '../request/types/ipTable'; export function isSupportIpTablePlatform() { return platformEnv.isNative || platformEnv.isDesktop; } -export async function verifyIpTableConfigSignature( - config: IIpTableRemoteConfig, -): Promise { - try { - const { signature, ...dataToVerify } = config; +/** + * Fields covered by the CDN signature. Extra metadata fields the pipeline may + * add later must NOT break verification, so we canonicalize an explicit + * allowlist instead of spreading "everything except signature". + */ +function buildCanonicalSignedPayload(config: IIpTableRemoteConfig): string { + return stringify({ + version: config.version, + ttl_sec: config.ttl_sec, + generated_at: config.generated_at, + domains: config.domains, + }); +} - if (!signature) { - console.error( - '[IpTableUtils] Signature verification failed: Missing signature', - ); +/** FNV-1a 32-bit — correlation id for logs/persistence, not a security hash. */ +export function computeIpTableConfigHash(config: IIpTableRemoteConfig): string { + const value = buildCanonicalSignedPayload(config); + let hash = 0x81_1c_9d_c5; + for (let i = 0; i < value.length; i += 1) { + hash ^= value.charCodeAt(i); + hash = Math.imul(hash, 0x01_00_01_93) >>> 0; + } + return hash.toString(16).padStart(8, '0'); +} + +/** + * Structural guard for CDN payloads. A misconfigured edge can return HTML or + * a truncated body that still parses; reject it before verification so the + * failure reason points at the delivery layer, not the signature. + */ +export function isValidIpTableRemoteConfigShape( + data: unknown, +): data is IIpTableRemoteConfig { + if (!data || typeof data !== 'object') { + return false; + } + const config = data as Partial; + if (typeof config.version !== 'number') { + return false; + } + if (typeof config.ttl_sec !== 'number') { + return false; + } + if (typeof config.generated_at !== 'string') { + return false; + } + if (typeof config.signature !== 'string') { + return false; + } + if (!config.domains || typeof config.domains !== 'object') { + return false; + } + for (const domainConfig of Object.values(config.domains)) { + if (!domainConfig || typeof domainConfig !== 'object') { return false; } + const endpoints = (domainConfig as { endpoints?: unknown }).endpoints; + if (!Array.isArray(endpoints)) { + return false; + } + for (const endpoint of endpoints) { + if (!endpoint || typeof endpoint !== 'object') { + return false; + } + if (typeof (endpoint as { ip?: unknown }).ip !== 'string') { + return false; + } + } + } + return true; +} - const canonicalString = stringify(dataToVerify); +function logVerifyEvent( + level: 'info' | 'error', + fields: Record, +): void { + const info = `[IpTableUtils] ${Object.entries({ + event: 'iptable_config_verify', + ...fields, + }) + .map(([key, value]) => `${key}=${String(value)}`) + .join(' ')}`; + if (level === 'error') { + defaultLogger.ipTable.request.error({ info }); + } else { + defaultLogger.ipTable.request.info({ info }); + } +} - const { verifyMessage } = await import('@ethersproject/wallet'); - const recoveredAddress = verifyMessage(canonicalString, signature); +/** + * Verify the CDN config signature with a structured result that separates + * the failure stages. `verifier_load_failed` means the crypto verifier could + * not even be loaded in this runtime (e.g. a split-bundle segment refused for + * the background runtime — the 2026-07-16 incident); it is a packaging + * problem, NOT an invalid signature, and must never be reported as one. + */ +export async function verifyIpTableConfigSignatureDetailed( + config: IIpTableRemoteConfig, +): Promise { + const payloadHash = computeIpTableConfigHash(config); + + if (!config.signature) { + logVerifyEvent('error', { + stage: 'precheck', + reason: 'missing_signature', + configVersion: config.version, + payloadHash, + }); + return { ok: false, reason: 'missing_signature' }; + } - const isValid = - recoveredAddress.toLowerCase() === CDN_SIGNER_ADDRESS.toLowerCase(); + const canonicalString = buildCanonicalSignedPayload(config); - if (!isValid) { - console.error( - '[IpTableUtils] Signature verification failed: Invalid signer', - '\n Expected:', - CDN_SIGNER_ADDRESS, - '\n Recovered:', - recoveredAddress, - ); - } + let verifyMessage: (message: string, signature: string) => string; + try { + ({ verifyMessage } = await import('@ethersproject/wallet')); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logVerifyEvent('error', { + stage: 'load_verifier', + reason: 'verifier_load_failed', + configVersion: config.version, + payloadHash, + errorMessage, + }); + return { ok: false, reason: 'verifier_load_failed', errorMessage }; + } - return isValid; + try { + const recoveredAddress = verifyMessage(canonicalString, config.signature); + if (recoveredAddress.toLowerCase() === CDN_SIGNER_ADDRESS.toLowerCase()) { + logVerifyEvent('info', { + stage: 'compare_signer', + result: 'ok', + configVersion: config.version, + generatedAt: config.generated_at, + payloadHash, + }); + return { ok: true, recoveredAddress }; + } + // Signer addresses are public constants; logging them is safe and is the + // single most useful datum when diagnosing a bad CDN publish. + logVerifyEvent('error', { + stage: 'compare_signer', + reason: 'signer_mismatch', + configVersion: config.version, + payloadHash, + recovered: recoveredAddress, + expected: CDN_SIGNER_ADDRESS, + }); + return { ok: false, reason: 'signer_mismatch' }; } catch (error) { - console.error('[IpTableUtils] Signature verification error:', error); - return false; + const errorMessage = error instanceof Error ? error.message : String(error); + logVerifyEvent('error', { + stage: 'recover', + reason: 'malformed_signature', + configVersion: config.version, + payloadHash, + errorMessage, + }); + return { ok: false, reason: 'malformed_signature', errorMessage }; } } +export async function verifyIpTableConfigSignature( + config: IIpTableRemoteConfig, +): Promise { + const result = await verifyIpTableConfigSignatureDetailed(config); + return result.ok; +} + export function mergeIpTableConfigs( localConfig: IIpTableRemoteConfig, remoteConfig: IIpTableRemoteConfig, From 4a78a799ca1e00fce7fcd5195427352fac02fdfd Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 20:04:41 +0800 Subject: [PATCH 07/22] feat: persist verified config provenance and reject version rollback --- .../simple/entity/SimpleDbEntityIpTable.ts | 17 +++++++++++- .../kit-bg/src/services/ServiceIpTable.ts | 27 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts index 5277a4500fdf..a48575e577a9 100644 --- a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts +++ b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts @@ -60,7 +60,10 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase { + async saveConfig( + config: IIpTableRemoteConfig, + verifiedMeta?: { payloadHash: string }, + ): Promise { await this.setRawData((data) => ({ ...data, config, @@ -72,6 +75,18 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase Date: Thu, 16 Jul 2026 20:06:41 +0800 Subject: [PATCH 08/22] feat: fetch ip table cdn config through iptable-capable client --- packages/kit-bg/src/services/ServiceIpTable.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 52f6e363fb54..2af626c5aea7 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -24,6 +24,7 @@ import { IP_TABLE_SPEED_TEST_TIMEOUT_MS, } from '@onekeyhq/shared/src/request/constants/ipTableDefaults'; import { + createAxiosWithIpTable, getSelectedIpForHost, setReportRequestFailureCallback, testDomainSpeed, @@ -297,7 +298,13 @@ class ServiceIpTable extends ServiceBase { info: `[IpTable] Fetching remote config from: ${IP_TABLE_CDN_URL}`, }); - const plainAxios = axios.create(); + // Fetch the config through the IP-Table-capable client: the config CDN + // (config.onekeycn.com) shares the walled root domain, so this fetch + // needs the same fail-open protection as business traffic. No cycle: + // endpoint selection only reads the local simpleDb/builtin table. + const plainAxios = createAxiosWithIpTable({ + timeout: IP_TABLE_CDN_FETCH_TIMEOUT_MS, + }); const headers = await getRequestHeaders(); From 7c10819d50b160fb9422741da3916bdd3c29a5ff Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 20:10:52 +0800 Subject: [PATCH 09/22] feat: server metrics for ip table failover and verify failures --- .../kit-bg/src/services/ServiceIpTable.ts | 29 ++++++++++++ .../shared/src/logger/scopes/ipTable/index.ts | 3 ++ .../logger/scopes/ipTable/scenes/metrics.ts | 44 +++++++++++++++++++ .../request/helpers/ipTableAdapter.test.ts | 5 +++ .../src/request/helpers/ipTableAdapter.ts | 8 ++++ 5 files changed, 89 insertions(+) create mode 100644 packages/shared/src/logger/scopes/ipTable/scenes/metrics.ts diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 2af626c5aea7..9d7653dcd750 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -385,6 +385,11 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.error({ info: `[IpTable] Skipping CDN config update: signature verification failed, reason=${verifyResult.reason}`, }); + defaultLogger.ipTable.metrics.configVerifyFailed({ + reason: verifyResult.reason, + configVersion: remoteConfig.version, + payloadHash: computeIpTableConfigHash(remoteConfig), + }); return false; } @@ -677,8 +682,26 @@ class ServiceIpTable extends ServiceBase { domain, decision.ip, ); + if ((currentSelection ?? '') === '') { + defaultLogger.ipTable.metrics.endpointSwitched({ + domain, + from: 'domain', + to: 'ip', + trigger: 'speed_test', + reason: decision.reason, + }); + } } else if (decision.action === 'select_domain') { await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ''); + if (currentSelection) { + defaultLogger.ipTable.metrics.endpointSwitched({ + domain, + from: 'ip', + to: 'domain', + trigger: 'speed_test', + reason: decision.reason, + }); + } } // no_change: keep the previous selection untouched. @@ -782,6 +805,12 @@ class ServiceIpTable extends ServiceBase { domain, lastBestIp, ); + defaultLogger.ipTable.metrics.endpointSwitched({ + domain, + from: 'domain', + to: 'ip', + trigger: 'fast_failover', + }); } } diff --git a/packages/shared/src/logger/scopes/ipTable/index.ts b/packages/shared/src/logger/scopes/ipTable/index.ts index 654d31ea4890..0856819e16ef 100644 --- a/packages/shared/src/logger/scopes/ipTable/index.ts +++ b/packages/shared/src/logger/scopes/ipTable/index.ts @@ -1,10 +1,13 @@ import { BaseScope } from '../../base/baseScope'; import { EScopeName } from '../../types'; +import { IpTableMetricsScene } from './scenes/metrics'; import { RequestScene } from './scenes/request'; export class IpTableScope extends BaseScope { protected override scopeName = EScopeName.ipTable; request = this.createScene('request', RequestScene); + + metrics = this.createScene('metrics', IpTableMetricsScene); } diff --git a/packages/shared/src/logger/scopes/ipTable/scenes/metrics.ts b/packages/shared/src/logger/scopes/ipTable/scenes/metrics.ts new file mode 100644 index 000000000000..80c8dc962e82 --- /dev/null +++ b/packages/shared/src/logger/scopes/ipTable/scenes/metrics.ts @@ -0,0 +1,44 @@ +import { BaseScene } from '../../../base/baseScene'; +import { LogToLocal, LogToServer } from '../../../base/decorators'; + +/** + * IP Table reliability metrics for server-side monitoring. + * + * These events exist so that endpoint failover behavior and config + * verification failures are visible without asking users to upload logs + * (the 2026-07-16 incident took a manual 11MB log upload to diagnose). + * + * No PII: only root domains, decision enums and short correlation hashes. + */ +export class IpTableMetricsScene extends BaseScene { + @LogToServer() + @LogToLocal() + public endpointSwitched(params: { + domain: string; + from: 'domain' | 'ip'; + to: 'domain' | 'ip'; + trigger: 'fast_failover' | 'speed_test'; + reason?: string; + }) { + return params; + } + + @LogToServer() + @LogToLocal() + public adapterFailover(params: { + lookupDomain: string; + action: 'activated' | 'deactivated'; + }) { + return params; + } + + @LogToServer() + @LogToLocal() + public configVerifyFailed(params: { + reason: string; + configVersion?: number; + payloadHash?: string; + }) { + return params; + } +} diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index 024a35edd978..5f688cd8cfd1 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -32,6 +32,11 @@ jest.mock('../../logger/logger', () => ({ warn: jest.fn(), error: jest.fn(), }, + metrics: { + endpointSwitched: jest.fn(), + adapterFailover: jest.fn(), + configVerifyFailed: jest.fn(), + }, }, }, })); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 79dc3e5a8c8f..d9f3f71770b8 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -254,6 +254,10 @@ async function recordDomainRequestOutcome(options: { lookupDomain, cause: 'domain_recovered', }); + defaultLogger.ipTable.metrics.adapterFailover({ + lookupDomain, + action: 'deactivated', + }); } state.consecutiveNetworkFailures = 0; state.failOpenUntil = 0; @@ -282,6 +286,10 @@ async function recordDomainRequestOutcome(options: { consecutiveNetworkFailures: state.consecutiveNetworkFailures, ttlMs: IP_TABLE_ADAPTER_FAILOVER_TTL_MS, }); + defaultLogger.ipTable.metrics.adapterFailover({ + lookupDomain, + action: 'activated', + }); } } From 12f24d9690c6947031c39995919e432091559b11 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 16 Jul 2026 20:12:39 +0800 Subject: [PATCH 10/22] feat: dev settings kill switch ui for ip table failover --- .../Setting/pages/Tab/DevSettingsSection/index.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx b/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx index 3cadc2699c47..26ac11d57e78 100644 --- a/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx +++ b/packages/kit/src/views/Setting/pages/Tab/DevSettingsSection/index.tsx @@ -1300,6 +1300,18 @@ const BaseDevSettingsSection = () => { > + + + Date: Fri, 17 Jul 2026 11:07:19 +0800 Subject: [PATCH 11/22] fix: classify transport errors per hostname and gate sni fallback by idempotency --- .../request/helpers/ipTableAdapter.test.ts | 321 ++++++++++++++ .../src/request/helpers/ipTableAdapter.ts | 395 +++++++++++++----- 2 files changed, 610 insertions(+), 106 deletions(-) diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index 5f688cd8cfd1..fc5985a7c07f 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -1,5 +1,6 @@ import axios from 'axios'; +import { OneKeyLocalError } from '../../errors'; import { getRequestHeaders } from '../Interceptor'; import requestHelper from '../requestHelper'; @@ -458,4 +459,324 @@ describe('ipTableAdapter fail-open on domain network failures', () => { ).resolves.toMatchObject({ data: { fallback: true } }); expect(mockedSniRequest).not.toHaveBeenCalled(); }); + + test('flipping the kill switch while fail-open is active takes effect immediately', async () => { + await failNTimes(3); + + // Circuit is open now; turn the kill switch on afterwards. + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + enabled: true, + settings: { disableIpTableFailover: true }, + } as never); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('cancellations do not count as transport failures', async () => { + fallbackAdapter.mockImplementation(async () => { + throw Object.assign(new Error('canceled'), { code: 'ERR_CANCELED' }); + }); + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ERR_CANCELED' }); + } + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('errors without an allowlisted transport code do not count', async () => { + fallbackAdapter.mockImplementation(async () => { + throw Object.assign( + new OneKeyLocalError('something exploded internally'), + { + code: 'SOME_INTERNAL_ERROR', + }, + ); + }); + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ message: 'something exploded internally' }); + } + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + + test('a domain success on another hostname does not close the circuit', async () => { + // wallet.* opens the circuit. + await failNTimes(3); + + // While the circuit is open, utility.* goes via SNI too; make its SNI + // attempt fail ambiguously (GET is idempotent -> falls back to domain) + // and let the domain succeed. That domain success comes from a hostname + // that did NOT open the circuit, so the circuit must stay open. + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + mockedSniRequest.mockRejectedValueOnce( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildConfig('https://utility.onekeycn.com/utility/v1/something'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + + // wallet traffic must still route via SNI (circuit still open). + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ status: 200, data: { ok: true } }); + const lastSniCall = mockedSniRequest.mock.calls.at(-1)?.[0] as { + hostname: string; + }; + expect(lastSniCall.hostname).toBe('wallet.onekeycn.com'); + }); + + test('a late success from a request started before activation does not close the circuit', async () => { + // One shared implementation routed by URL marker so the in-flight stale + // request is unaffected when the failing behavior is exercised. + let resolveStale: (() => void) | undefined; + const stalePending = new Promise((resolve) => { + resolveStale = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('stale-probe')) { + await stalePending; + return { + data: { stale: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + const staleRequest = createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/stale-probe'), + ); + // Ensure the stale request's transport start timestamp precedes the + // activation timestamp deterministically. + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // Open the circuit with 3 fresh transport failures. + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // Let the stale request resolve successfully AFTER activation. + resolveStale?.(); + await expect(staleRequest).resolves.toMatchObject({ + data: { stale: true }, + }); + + // Circuit must still be open: next request goes via SNI. + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ status: 200, data: { ok: true } }); + expect(mockedSniRequest).toHaveBeenCalledTimes(1); + }); +}); + +describe('ipTableAdapter idempotency-gated fallback after SNI started', () => { + let originalAdapter: typeof axios.defaults.adapter; + let fallbackAdapter: jest.Mock< + Promise, + [InternalAxiosRequestConfig] + >; + let consoleErrorSpy: jest.SpyInstance; + + function buildMethodConfig( + url: string, + method: string, + ): InternalAxiosRequestConfig { + return { + url, + method, + headers: axios.AxiosHeaders.from({}), + } as InternalAxiosRequestConfig; + } + + beforeEach(() => { + consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + originalAdapter = axios.defaults.adapter; + fallbackAdapter = jest.fn(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + axios.defaults.adapter = fallbackAdapter; + + mockedIsSniSupported.mockReturnValue(true); + mockedIsProxyActiveForUrl.mockResolvedValue(false); + mockedSniRequest.mockReset(); + mockedGetRequestHeaders.mockResolvedValue({}); + mockedRequestHelper.getDevSettingsPersistAtom.mockResolvedValue({ + settings: {}, + } as never); + mockedRequestHelper.getIpTableConfig.mockResolvedValue({ + config: { + version: 1, + ttl_sec: 60, + generated_at: '2026-06-30T00:00:00.000Z', + signature: '', + domains: { + 'example.com': { + endpoints: [ + { + ip: '93.184.216.34', + provider: 'test', + region: 'ALL', + weight: 1, + }, + ], + }, + }, + }, + runtime: { + enabled: true, + lastUpdated: 0, + lastRegionCheck: 0, + selections: { + 'example.com': '93.184.216.34', + }, + }, + } as never); + resetAdapterFailoverStatesForTesting(); + }); + + afterEach(() => { + axios.defaults.adapter = originalAdapter; + consoleErrorSpy.mockRestore(); + resetAdapterFailoverStatesForTesting(); + jest.clearAllMocks(); + }); + + test('GET falls back to domain after an ambiguous SNI timeout', async () => { + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'get'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(fallbackAdapter).toHaveBeenCalledTimes(1); + }); + + test('POST does NOT fall back after an ambiguous SNI timeout (double-send risk)', async () => { + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ code: 'SNI_TIMEOUT' }); + expect(fallbackAdapter).not.toHaveBeenCalled(); + }); + + test('POST falls back when the error proves the connection was never established', async () => { + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('connection refused'), { + code: 'SNI_CONNECTION_REFUSED', + }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(fallbackAdapter).toHaveBeenCalledTimes(1); + }); + + test('POST does NOT fall back when SNI returns a null response', async () => { + mockedSniRequest.mockResolvedValue(null); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ + message: expect.stringContaining('not idempotent'), + }); + expect(fallbackAdapter).not.toHaveBeenCalled(); + }); }); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index d9f3f71770b8..74bd4c1c3e4d 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -111,6 +111,52 @@ let reportRequestFailureCallback: | ((params: IRequestFailureParams) => void) | null = null; +/** + * Request success callback parameters. Successes must flow to the service so + * its consecutive-failure counters reset on real recovery instead of only + * ever incrementing. + */ +interface IRequestSuccessParams { + domain: string; + requestType: 'ip' | 'domain'; + target: string; +} + +let reportRequestSuccessCallback: + | ((params: IRequestSuccessParams) => void) + | null = null; + +export function setReportRequestSuccessCallback( + callback: (params: IRequestSuccessParams) => void, +) { + reportRequestSuccessCallback = callback; +} + +/** + * Once a request has been handed to the SNI transport it may have reached + * the server even when we got no usable response back. Re-sending it over + * the domain is only safe for idempotent methods, or when the error code + * proves the connection was never established (nothing was written). + */ +const IDEMPOTENT_HTTP_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']); + +const SNI_PRE_WRITE_ERROR_CODES = new Set([ + 'SNI_CONNECTION_REFUSED', + 'SNI_DNS_FAILED', + 'SNI_NETWORK_UNREACHABLE', + 'SNI_INVALID_URL', +]); + +function canFallbackAfterSniStarted(method: string, error?: unknown): boolean { + if (IDEMPOTENT_HTTP_METHODS.has(method.toUpperCase())) { + return true; + } + if (error && SNI_PRE_WRITE_ERROR_CODES.has(getErrorCode(error))) { + return true; + } + return false; +} + /** * Extract root domain from hostname * Example: wallet.example.com -> example.com @@ -167,18 +213,30 @@ const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { // ========== Fail-open on domain network failures ========== // // When real requests on the direct domain keep failing at the transport -// level (timeouts, resets — NOT HTTP error responses), the adapter opens a -// time-boxed window during which hosts of that root domain resolve to a -// fallback IP (runtime last-best IP when available, builtin config -// otherwise). This is the only protection available in runtimes where the -// simpleDb-backed config is absent: the main runtime on split-runtime +// level (timeouts, resets — NOT HTTP error responses or cancellations), the +// adapter opens a time-boxed window during which hosts of that root domain +// resolve to a fallback IP (runtime last-best IP when available, builtin +// config otherwise). This is the only protection available in runtimes where +// the simpleDb-backed config is absent: the main runtime on split-runtime // targets and the cold-start window before background init completes. -// Failing requests are never replayed — a wallet must not double-send -// mutating POSTs — only subsequent requests take the fail-open route. +// The request that trips the threshold is never replayed — only subsequent +// requests take the fail-open route — and once a request has been handed to +// the SNI transport, falling back to the domain (which re-sends it) is +// restricted to idempotent methods or provably-unsent errors; see +// canFallbackAfterSniStarted. + +interface IHostFailureRecord { + consecutive: number; + lastFailureAt: number; +} interface IAdapterFailoverState { - consecutiveNetworkFailures: number; + /** Consecutive transport failures tracked per hostname, not per root domain */ + hostFailures: Map; failOpenUntil: number; + activatedAt: number; + /** Hostnames whose failures opened the circuit; only they may close it early */ + activatedHostnames: Set; /** Rotates across activations so a dead builtin endpoint is not retried forever */ fallbackIpIndex: number; } @@ -191,16 +249,36 @@ export function resetAdapterFailoverStatesForTesting(): void { getSelectedIpForHost.clear(); } -function isNetworkLevelError(error: unknown): boolean { +/** + * Transport-level failure allowlist. Only errors that prove the network path + * itself is broken may open the circuit. Cancellations (AbortController, + * axios dedup, page unload) and HTTP error responses must never count: a + * cancel says nothing about the network, and a response proves it works. + */ +const TRANSPORT_ERROR_CODES = new Set([ + 'ERR_NETWORK', // axios v1 generic network failure (xhr/fetch) + 'ECONNABORTED', // axios timeout + 'ETIMEDOUT', + 'ECONNRESET', + 'ECONNREFUSED', + 'ENOTFOUND', + 'ENETUNREACH', + 'EHOSTUNREACH', + 'EAI_AGAIN', +]); + +function isTransportLevelError(error: unknown): boolean { if (!error || typeof error !== 'object') { return false; } - // An HTTP response means the network path works; only transport-level - // failures (timeout, reset, dns) should trigger fail-open. + if (axios.isCancel(error)) { + return false; + } if ('response' in error && (error as { response?: unknown }).response) { return false; } - return true; + const code = getErrorCode(error); + return TRANSPORT_ERROR_CODES.has(code); } async function isFailoverDisabledByDevSettings(): Promise { @@ -217,8 +295,10 @@ function getFailoverState(lookupDomain: string): IAdapterFailoverState { let state = adapterFailoverStates.get(lookupDomain); if (!state) { state = { - consecutiveNetworkFailures: 0, + hostFailures: new Map(), failOpenUntil: 0, + activatedAt: 0, + activatedHostnames: new Set(), fallbackIpIndex: 0, }; adapterFailoverStates.set(lookupDomain, state); @@ -231,6 +311,28 @@ function isFailoverActive(lookupDomain: string): boolean { return Boolean(state && state.failOpenUntil > Date.now()); } +function deactivateFailover(lookupDomain: string, cause: string): void { + const state = adapterFailoverStates.get(lookupDomain); + if (!state) { + return; + } + if (state.failOpenUntil > Date.now()) { + logIpTableEvent('info', 'adapter_failover_deactivated', { + lookupDomain, + cause, + }); + defaultLogger.ipTable.metrics.adapterFailover({ + lookupDomain, + action: 'deactivated', + }); + } + state.hostFailures.clear(); + state.failOpenUntil = 0; + state.activatedAt = 0; + state.activatedHostnames.clear(); + getSelectedIpForHost.clear(); +} + async function resolveFailoverLookupDomain( rootDomain: string, ): Promise { @@ -240,56 +342,69 @@ async function resolveFailoverLookupDomain( async function recordDomainRequestOutcome(options: { rootDomain: string; + hostname: string; ok: boolean; + /** Date.now() captured when the request was handed to the transport */ + startedAtMs: number; error?: unknown; }): Promise { - const { rootDomain, ok, error } = options; + const { rootDomain, hostname, ok, startedAtMs, error } = options; const lookupDomain = await resolveFailoverLookupDomain(rootDomain); const state = getFailoverState(lookupDomain); if (ok) { - if (state.consecutiveNetworkFailures > 0 || state.failOpenUntil > 0) { - if (state.failOpenUntil > Date.now()) { - logIpTableEvent('info', 'adapter_failover_deactivated', { - lookupDomain, - cause: 'domain_recovered', - }); - defaultLogger.ipTable.metrics.adapterFailover({ - lookupDomain, - action: 'deactivated', - }); - } - state.consecutiveNetworkFailures = 0; - state.failOpenUntil = 0; - getSelectedIpForHost.clear(); + const hostFailure = state.hostFailures.get(hostname); + // A success only proves the path that was exercised: it resets its own + // hostname's counter, and it may close the circuit only when it comes + // from a hostname that opened it AND started after activation (a + // long-in-flight request resolving late says nothing about recovery). + if (hostFailure && startedAtMs >= hostFailure.lastFailureAt) { + state.hostFailures.delete(hostname); + } + if ( + state.failOpenUntil > Date.now() && + state.activatedHostnames.has(hostname) && + startedAtMs >= state.activatedAt + ) { + deactivateFailover(lookupDomain, 'domain_recovered'); } return; } - if (!isNetworkLevelError(error)) { + if (!isTransportLevelError(error)) { return; } if (await isFailoverDisabledByDevSettings()) { return; } - state.consecutiveNetworkFailures += 1; - if ( - state.consecutiveNetworkFailures >= IP_TABLE_DOMAIN_FAILOVER_THRESHOLD && - state.failOpenUntil <= Date.now() - ) { - state.failOpenUntil = Date.now() + IP_TABLE_ADAPTER_FAILOVER_TTL_MS; - state.fallbackIpIndex += 1; - getSelectedIpForHost.clear(); - logIpTableEvent('warn', 'adapter_failover_activated', { - lookupDomain, - consecutiveNetworkFailures: state.consecutiveNetworkFailures, - ttlMs: IP_TABLE_ADAPTER_FAILOVER_TTL_MS, - }); - defaultLogger.ipTable.metrics.adapterFailover({ - lookupDomain, - action: 'activated', - }); + const now = Date.now(); + const hostFailure = state.hostFailures.get(hostname) ?? { + consecutive: 0, + lastFailureAt: 0, + }; + hostFailure.consecutive += 1; + hostFailure.lastFailureAt = now; + state.hostFailures.set(hostname, hostFailure); + + if (hostFailure.consecutive >= IP_TABLE_DOMAIN_FAILOVER_THRESHOLD) { + state.activatedHostnames.add(hostname); + if (state.failOpenUntil <= now) { + state.failOpenUntil = now + IP_TABLE_ADAPTER_FAILOVER_TTL_MS; + state.activatedAt = now; + state.fallbackIpIndex += 1; + getSelectedIpForHost.clear(); + logIpTableEvent('warn', 'adapter_failover_activated', { + lookupDomain, + hostname, + consecutiveNetworkFailures: hostFailure.consecutive, + ttlMs: IP_TABLE_ADAPTER_FAILOVER_TTL_MS, + }); + defaultLogger.ipTable.metrics.adapterFailover({ + lookupDomain, + action: 'activated', + }); + } } } @@ -303,13 +418,23 @@ function getFailoverFallbackIp(lookupDomain: string): string | null { } /** - * Resolve the fail-open IP for a lookup domain, preferring the last-best IP - * measured by the background speed test when a runtime is available. + * Resolve the fail-open IP for a lookup domain when the circuit is open. + * Prefers the last-best IP measured by the background speed test when a + * runtime is available. Honors the kill switch immediately: flipping + * `disableIpTableFailover` while a circuit is open clears the state instead + * of waiting for the TTL to expire. */ -function resolveFailoverIp( +async function getActiveFailoverIp( lookupDomain: string, runtimeLastBestIp: string | undefined, -): string | null { +): Promise { + if (!isFailoverActive(lookupDomain)) { + return null; + } + if (await isFailoverDisabledByDevSettings()) { + deactivateFailover(lookupDomain, 'kill_switch'); + return null; + } return runtimeLastBestIp || getFailoverFallbackIp(lookupDomain); } @@ -386,21 +511,19 @@ async function getSelectedIpForHostInternal( // too. runtime.enabled === false is explicit user intent: no fail-open. if (!configWithRuntime) { const lookupDomain = await resolveFailoverLookupDomain(rootDomain); - if (isFailoverActive(lookupDomain)) { - const failoverIp = resolveFailoverIp(lookupDomain, undefined); - if (failoverIp) { - logIpTableEvent('warn', 'iptable_selection', { - hostname, - rootDomain, - lookupDomain, - mapped: lookupDomain !== rootDomain, - strictMode: false, - runtimeEnabled: false, - decision: 'fail_open', - selectedIpHash: hashForLog(failoverIp), - }); - return failoverIp; - } + const failoverIp = await getActiveFailoverIp(lookupDomain, undefined); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, + lookupDomain, + mapped: lookupDomain !== rootDomain, + strictMode: false, + runtimeEnabled: false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; } } logIpTableEvent('info', 'iptable_selection', { @@ -456,24 +579,22 @@ async function getSelectedIpForHostInternal( // In strict mode, override this and use fallback IP from config if (selectedIp === '') { if (!strictMode) { - if (isFailoverActive(lookupDomain)) { - const failoverIp = resolveFailoverIp( + const failoverIp = await getActiveFailoverIp( + lookupDomain, + runtime?.lastBestIp?.[lookupDomain], + ); + if (failoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, lookupDomain, - runtime?.lastBestIp?.[lookupDomain], - ); - if (failoverIp) { - logIpTableEvent('warn', 'iptable_selection', { - hostname, - rootDomain, - lookupDomain, - mapped: Boolean(mappedDomain), - strictMode: false, - runtimeEnabled: runtime?.enabled !== false, - decision: 'fail_open', - selectedIpHash: hashForLog(failoverIp), - }); - return failoverIp; - } + mapped: Boolean(mappedDomain), + strictMode: false, + runtimeEnabled: runtime?.enabled !== false, + decision: 'fail_open', + selectedIpHash: hashForLog(failoverIp), + }); + return failoverIp; } debugLog( `[IpTableAdapter] Explicitly using domain for: ${lookupDomain}`, @@ -514,24 +635,22 @@ async function getSelectedIpForHostInternal( } } - if (isFailoverActive(lookupDomain)) { - const failoverIp = resolveFailoverIp( + const noSelectionFailoverIp = await getActiveFailoverIp( + lookupDomain, + runtime?.lastBestIp?.[lookupDomain], + ); + if (noSelectionFailoverIp) { + logIpTableEvent('warn', 'iptable_selection', { + hostname, + rootDomain, lookupDomain, - runtime?.lastBestIp?.[lookupDomain], - ); - if (failoverIp) { - logIpTableEvent('warn', 'iptable_selection', { - hostname, - rootDomain, - lookupDomain, - mapped: Boolean(mappedDomain), - strictMode: Boolean(strictMode), - runtimeEnabled: runtime?.enabled !== false, - decision: 'fail_open', - selectedIpHash: hashForLog(failoverIp), - }); - return failoverIp; - } + mapped: Boolean(mappedDomain), + strictMode: Boolean(strictMode), + runtimeEnabled: runtime?.enabled !== false, + decision: 'fail_open', + selectedIpHash: hashForLog(noSelectionFailoverIp), + }); + return noSelectionFailoverIp; } logIpTableEvent('info', 'iptable_selection', { @@ -684,6 +803,7 @@ export function createIpTableAdapter( rootDomain?: string; }): Promise => { const { config, isFallback = false, hostname, rootDomain } = options; + const startedAtMs = Date.now(); debugLog('[IpTableAdapter] About to call original adapter...'); debugLog( '[IpTableAdapter] Original adapter type:', @@ -740,27 +860,50 @@ export function createIpTableAdapter( ); } - if (rootDomain) { + if (rootDomain && hostname) { // A completed domain request (any HTTP status) proves the network - // path works: reset the adapter fail-open counter. - await recordDomainRequestOutcome({ rootDomain, ok: true }); + // path works for THIS hostname: reset its fail-open counter and + // notify the service so its health stats reset too. + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: true, + startedAtMs, + }); + if (reportRequestSuccessCallback) { + reportRequestSuccessCallback({ + domain: rootDomain, + requestType: 'domain', + target: hostname, + }); + } } return response; } catch (error) { - if (rootDomain) { + if (rootDomain && hostname) { // Track transport-level failures for adapter fail-open. The failing // request itself is never replayed (no double-send of POSTs); only // subsequent requests take the fail-open route. - await recordDomainRequestOutcome({ rootDomain, ok: false, error }); + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: false, + startedAtMs, + error, + }); } - // Only report domain failures if this is NOT a fallback request + // Only report domain failures if this is NOT a fallback request, and + // only for transport-level failures: HTTP responses prove the network + // path works and cancellations say nothing about it, so neither may + // drive the service's failover/speed-test counters. if ( !isFallback && hostname && rootDomain && - reportRequestFailureCallback + reportRequestFailureCallback && + isTransportLevelError(error) ) { debugLog( `[IpTableAdapter] Domain request failed (not fallback): ${hostname}`, @@ -1038,6 +1181,21 @@ export function createIpTableAdapter( error: 'SNI response null', }); } + // A null response is ambiguous — the request may have reached the + // server. Re-sending over the domain is only safe for idempotent + // methods; anything else must surface the failure to the caller. + const method = (config.method || 'GET').toUpperCase(); + if (!canFallbackAfterSniStarted(method)) { + logIpTableEvent('warn', 'sni_fallback_blocked', { + hostname, + rootDomain, + method, + reason: 'non_idempotent_after_sni_started', + }); + throw new OneKeyLocalError( + 'IP Table Adapter: SNI response missing and request is not idempotent', + ); + } // Fallback to domain (isFallback = true, so domain failure won't be counted) return await callOriginalAdapter({ config, @@ -1052,6 +1210,14 @@ export function createIpTableAdapter( `[IpTableAdapter] SNI request successful: ${sniResponse.statusCode}`, ); + if (reportRequestSuccessCallback) { + reportRequestSuccessCallback({ + domain: rootDomain, + requestType: 'ip', + target: selectedIp, + }); + } + // Parse response body let responseData: any = null; const responseBody = sniResponse.body ?? sniResponse.data; @@ -1111,6 +1277,23 @@ export function createIpTableAdapter( }); } + // Once the request was handed to the SNI transport it may have been + // written to the wire (e.g. SNI_TIMEOUT after send). Re-sending over + // the domain risks double-executing mutating requests, so fallback is + // restricted to idempotent methods or errors that prove the + // connection was never established. + const method = (config.method || 'GET').toUpperCase(); + if (!canFallbackAfterSniStarted(method, error)) { + logIpTableEvent('warn', 'sni_fallback_blocked', { + hostname, + rootDomain, + method, + errorCode: getErrorCode(error), + reason: 'non_idempotent_after_sni_started', + }); + throw error; + } + // If SNI request throws error, use original adapter debugWarn( '[IpTableAdapter] SNI request failed, falling back to original adapter:', From 459abea292fbab349a7e9c7d426a5b526d72e162 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 11:09:57 +0800 Subject: [PATCH 12/22] fix: escalate coalesced domain-failure speed tests and guard stale selection writes --- .../kit-bg/src/services/ServiceIpTable.ts | 126 +++++++++++++++--- 1 file changed, 105 insertions(+), 21 deletions(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 9d7653dcd750..0d9fd0a9eb9c 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -27,6 +27,7 @@ import { createAxiosWithIpTable, getSelectedIpForHost, setReportRequestFailureCallback, + setReportRequestSuccessCallback, testDomainSpeed, testIpSpeed, } from '@onekeyhq/shared/src/request/helpers/ipTableAdapter'; @@ -539,7 +540,30 @@ class ServiceIpTable extends ServiceBase { // Coalesce concurrent speed tests per domain (anti probe-storm): the log // of the 2026-07-16 incident shows ~20 concurrent rounds saturating an // already congested network and polluting the latency measurements. - private speedTestInFlight = new Map>(); + private speedTestInFlight = new Map< + string, + { + promise: Promise; + trigger: 'domain_failure' | 'ip_failure' | 'periodic'; + } + >(); + + // Monotonic per-domain token bumped on every selection write. A speed test + // captures it at start and must not apply a stale result if someone else + // (e.g. fast failover) wrote a selection while it was probing. + private selectionGeneration = new Map(); + + private bumpSelectionGeneration(domain: string): void { + this.selectionGeneration.set( + domain, + (this.selectionGeneration.get(domain) ?? 0) + 1, + ); + } + + private async applySelection(domain: string, ip: string): Promise { + this.bumpSelectionGeneration(domain); + await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ip); + } /** * Select best endpoint for a domain @@ -548,6 +572,11 @@ class ServiceIpTable extends ServiceBase { * real-traffic domain failures, any reachable IP wins regardless of the * latency threshold. */ + // Domains that reported real-traffic domain failures while a weaker-trigger + // speed test was in flight; a follow-up domain_failure round runs for them + // so the reachability-first escalation is never silently coalesced away. + private pendingDomainFailureRerun = new Set(); + @backgroundMethod() async selectBestEndpointForDomain( domain: string, @@ -555,18 +584,39 @@ class ServiceIpTable extends ServiceBase { ): Promise { const inflight = this.speedTestInFlight.get(domain); if (inflight) { - defaultLogger.ipTable.request.info({ - info: `[IpTable] Speed test already in flight for ${domain}, coalescing`, - }); - return inflight; + // Do not let a stronger trigger be swallowed by a weaker in-flight + // run: the merged run would compute with domainFailingRealTraffic set + // to false and could keep a failing domain selected (the exact + // 2026-07-16 incident shape). Queue one follow-up round instead. + if ( + opts?.trigger === 'domain_failure' && + inflight.trigger !== 'domain_failure' + ) { + this.pendingDomainFailureRerun.add(domain); + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Speed test in flight for ${domain} (trigger=${inflight.trigger}); queueing domain_failure follow-up round`, + }); + } else { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Speed test already in flight for ${domain}, coalescing`, + }); + } + return inflight.promise; } - const run = this.selectBestEndpointForDomainInternal(domain, opts).finally( - () => { - this.speedTestInFlight.delete(domain); - }, - ); - this.speedTestInFlight.set(domain, run); - return run; + const trigger = opts?.trigger ?? 'periodic'; + const promise = this.selectBestEndpointForDomainInternal( + domain, + opts, + ).finally(() => { + this.speedTestInFlight.delete(domain); + if (this.pendingDomainFailureRerun.delete(domain)) { + void this.selectBestEndpointForDomain(domain, { + trigger: 'domain_failure', + }); + } + }); + this.speedTestInFlight.set(domain, { promise, trigger }); + return promise; } private async selectBestEndpointForDomainInternal( @@ -595,6 +645,11 @@ class ServiceIpTable extends ServiceBase { return; } + // A probing round can take tens of seconds; capture the selection + // generation so a stale result cannot overwrite a fast failover that + // happened while we were measuring. + const startGeneration = this.selectionGeneration.get(domain) ?? 0; + try { // 1. Test domain directly defaultLogger.ipTable.request.info({ @@ -677,11 +732,25 @@ class ServiceIpTable extends ServiceBase { }) trigger=${opts?.trigger ?? 'periodic'} domainLatency=${domainLatency}ms`, }); + // Compare-and-set: someone (fast failover, another round) wrote a + // selection while this round was probing. A weaker-trigger result + // computed from pre-write state must not clobber it; domain_failure + // results stay authoritative because they encode live traffic reality. + const currentGeneration = this.selectionGeneration.get(domain) ?? 0; + if ( + currentGeneration !== startGeneration && + opts?.trigger !== 'domain_failure' + ) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Discarding stale speed test result for ${domain}: selection changed while probing (trigger=${ + opts?.trigger ?? 'periodic' + })`, + }); + return; + } + if (decision.action === 'select_ip') { - await this.backgroundApi.simpleDb.ipTable.updateSelection( - domain, - decision.ip, - ); + await this.applySelection(domain, decision.ip); if ((currentSelection ?? '') === '') { defaultLogger.ipTable.metrics.endpointSwitched({ domain, @@ -692,7 +761,7 @@ class ServiceIpTable extends ServiceBase { }); } } else if (decision.action === 'select_domain') { - await this.backgroundApi.simpleDb.ipTable.updateSelection(domain, ''); + await this.applySelection(domain, ''); if (currentSelection) { defaultLogger.ipTable.metrics.endpointSwitched({ domain, @@ -801,10 +870,7 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.warn({ info: `[IpTable] Fast failover: domain ${domain} failing real traffic, switching to last-best IP immediately`, }); - await this.backgroundApi.simpleDb.ipTable.updateSelection( - domain, - lastBestIp, - ); + await this.applySelection(domain, lastBestIp); defaultLogger.ipTable.metrics.endpointSwitched({ domain, from: 'domain', @@ -909,6 +975,24 @@ class ServiceIpTable extends ServiceBase { void this.reportRequestFailure(domain, requestType, target); }); + // Successes reset the consecutive-failure counters, so "consecutive" + // means what it says: interleaved successes prevent sporadic errors + // accumulated over hours from ever tripping the failover threshold. + setReportRequestSuccessCallback(({ domain, requestType, target }) => { + const stats = this.domainHealthMap.get(domain); + if (!stats) { + return; + } + if (requestType === 'domain') { + stats.domainDirect.consecutiveFailures = 0; + } else { + const ipHealth = stats.ipEndpoints.get(target); + if (ipHealth) { + ipHealth.consecutiveFailures = 0; + } + } + }); + // Try to refresh CDN config if needed const shouldRefresh = await this.shouldRefreshConfig(); let needSpeedTest = false; From 6aaf609dc9c326d6a38322388ba5fd2d40b91f23 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 11:11:45 +0800 Subject: [PATCH 13/22] fix: anchor config rollback protection to the active config when unverified --- .../kit-bg/src/services/ServiceIpTable.ts | 29 ++++---- .../shared/src/utils/ipTableUtils.test.ts | 73 +++++++++++++++++++ packages/shared/src/utils/ipTableUtils.ts | 38 ++++++++++ 3 files changed, 124 insertions(+), 16 deletions(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 0d9fd0a9eb9c..4487566ead95 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -40,6 +40,7 @@ import type { } from '@onekeyhq/shared/src/request/types/ipTable'; import { computeIpTableConfigHash, + isIpTableConfigRegression, isSupportIpTablePlatform, isValidIpTableRemoteConfigShape, mergeIpTableConfigs, @@ -402,23 +403,19 @@ class ServiceIpTable extends ServiceBase { const localConfig = currentConfig.config; const lastVerified = currentConfig.runtime?.lastVerified; - // Rollback protection: a valid signature is not enough — refuse - // configs older than what we already verified, so a stale (but - // legitimately signed) file cannot displace a newer one. - const remoteGeneratedAtMs = Date.parse(remoteConfig.generated_at); - const lastVerifiedGeneratedAtMs = lastVerified - ? Date.parse(lastVerified.generatedAt) - : Number.NaN; - const isVersionRegression = - remoteConfig.version < localConfig.version || - (lastVerified !== undefined && - remoteConfig.version === lastVerified.version && - !Number.isNaN(remoteGeneratedAtMs) && - !Number.isNaN(lastVerifiedGeneratedAtMs) && - remoteGeneratedAtMs < lastVerifiedGeneratedAtMs); - if (isVersionRegression) { + // Rollback protection: refuse configs older than what we already + // verified. Falls back to the active config's own version/generated_at + // as the anchor when no lastVerified exists yet (pre-upgrade installs). + const regressionCheck = isIpTableConfigRegression({ + remoteConfig, + localConfig, + lastVerified, + }); + if (regressionCheck.regression) { defaultLogger.ipTable.request.error({ - info: `[IpTable] Rejecting CDN config rollback: remote v${remoteConfig.version}@${remoteConfig.generated_at} is older than local v${localConfig.version}`, + info: `[IpTable] Rejecting CDN config update: ${ + regressionCheck.reason ?? 'regression' + } (remote v${remoteConfig.version}@${remoteConfig.generated_at}, local v${localConfig.version})`, }); return false; } diff --git a/packages/shared/src/utils/ipTableUtils.test.ts b/packages/shared/src/utils/ipTableUtils.test.ts index 6b7b0b66c37a..45b04a1ec8b5 100644 --- a/packages/shared/src/utils/ipTableUtils.test.ts +++ b/packages/shared/src/utils/ipTableUtils.test.ts @@ -6,6 +6,7 @@ import { import { computeIpTableConfigHash, + isIpTableConfigRegression, isValidIpTableRemoteConfigShape, mergeIpTableConfigs, verifyIpTableConfigSignature, @@ -281,6 +282,78 @@ describe('isValidIpTableRemoteConfigShape', () => { }); }); +describe('isIpTableConfigRegression', () => { + function makeConfig(version: number, generatedAt: string) { + return { + ...DEFAULT_IP_TABLE_CONFIG, + version, + generated_at: generatedAt, + }; + } + + test('without lastVerified, the active config generated_at anchors same-version comparison', () => { + // Pre-upgrade installs have no runtime.lastVerified; a same-version but + // older signed config must still be rejected against the active config. + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2025-10-01T00:00:00.000Z'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + lastVerified: undefined, + }); + expect(result).toEqual({ + regression: true, + reason: 'generated_at_regression', + }); + }); + + test('same version with newer generated_at is accepted', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2026-01-01T00:00:00.000Z'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + }); + expect(result).toEqual({ regression: false }); + }); + + test('lower version is always a regression', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2026-01-01T00:00:00.000Z'), + localConfig: makeConfig(2, '2025-11-06T08:30:54.066Z'), + }); + expect(result).toEqual({ regression: true, reason: 'version_regression' }); + }); + + test('unparseable remote generated_at is rejected', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(2, 'not-a-date'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + }); + expect(result).toEqual({ + regression: true, + reason: 'unparseable_generated_at', + }); + }); + + test('lastVerified anchor takes precedence over the active config', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(1, '2025-12-01T00:00:00.000Z'), + localConfig: makeConfig(1, '2025-11-06T08:30:54.066Z'), + lastVerified: { version: 1, generatedAt: '2026-01-01T00:00:00.000Z' }, + }); + expect(result).toEqual({ + regression: true, + reason: 'generated_at_regression', + }); + }); + + test('higher version wins regardless of generated_at', () => { + const result = isIpTableConfigRegression({ + remoteConfig: makeConfig(3, '2025-01-01T00:00:00.000Z'), + localConfig: makeConfig(2, '2025-11-06T08:30:54.066Z'), + lastVerified: { version: 2, generatedAt: '2025-11-06T08:30:54.066Z' }, + }); + expect(result).toEqual({ regression: false }); + }); +}); + describe('computeIpTableConfigHash', () => { test('is stable and ignores unsigned extra fields', () => { const base = computeIpTableConfigHash(DEFAULT_IP_TABLE_CONFIG); diff --git a/packages/shared/src/utils/ipTableUtils.ts b/packages/shared/src/utils/ipTableUtils.ts index 819c3bbdace7..eee96efcbb6f 100644 --- a/packages/shared/src/utils/ipTableUtils.ts +++ b/packages/shared/src/utils/ipTableUtils.ts @@ -186,6 +186,44 @@ export async function verifyIpTableConfigSignature( return result.ok; } +/** + * Rollback protection: a valid signature is not enough — a stale but + * legitimately signed config must not displace a newer one. When no + * `lastVerified` anchor exists yet (installs that predate provenance + * tracking), the currently active config's own version/generated_at serves + * as the anchor, so the monotonic guarantee holds from the first update. + */ +export function isIpTableConfigRegression(options: { + remoteConfig: IIpTableRemoteConfig; + localConfig: IIpTableRemoteConfig; + lastVerified?: { version: number; generatedAt: string }; +}): { regression: boolean; reason?: string } { + const { remoteConfig, localConfig, lastVerified } = options; + + const remoteGeneratedAtMs = Date.parse(remoteConfig.generated_at); + if (Number.isNaN(remoteGeneratedAtMs)) { + return { regression: true, reason: 'unparseable_generated_at' }; + } + + if (remoteConfig.version < localConfig.version) { + return { regression: true, reason: 'version_regression' }; + } + + const anchorVersion = lastVerified?.version ?? localConfig.version; + const anchorGeneratedAtMs = Date.parse( + lastVerified?.generatedAt ?? localConfig.generated_at, + ); + if ( + remoteConfig.version === anchorVersion && + !Number.isNaN(anchorGeneratedAtMs) && + remoteGeneratedAtMs < anchorGeneratedAtMs + ) { + return { regression: true, reason: 'generated_at_regression' }; + } + + return { regression: false }; +} + export function mergeIpTableConfigs( localConfig: IIpTableRemoteConfig, remoteConfig: IIpTableRemoteConfig, From 41105111d757fc5acc31487c75ed47621a916467 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 11:13:06 +0800 Subject: [PATCH 14/22] fix: direct sni fallback for cdn config fetch on fresh installs --- .../kit-bg/src/services/ServiceIpTable.ts | 98 ++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 4487566ead95..134182029d84 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -32,7 +32,11 @@ import { testIpSpeed, } from '@onekeyhq/shared/src/request/helpers/ipTableAdapter'; import { decideEndpoint } from '@onekeyhq/shared/src/request/helpers/ipTableEndpointDecision'; -import { isSniSupported } from '@onekeyhq/shared/src/request/helpers/sniRequest'; +import { + isProxyActiveForUrl, + isSniSupported, + sniRequest, +} from '@onekeyhq/shared/src/request/helpers/sniRequest'; import { getRequestHeaders } from '@onekeyhq/shared/src/request/Interceptor'; import type { IIpTableConfigWithRuntime, @@ -361,6 +365,98 @@ class ServiceIpTable extends ServiceBase { }`, }); } + // The adapter's fail-open needs 3 accumulated transport failures in + // THIS runtime, which a fresh install's single config GET can never + // reach. The config GET is idempotent, so replaying it over SNI with + // known candidate IPs is safe and gives first-run devices on walled + // networks a working config path. + return this.fetchRemoteConfigViaSniFallback(); + } + } + + /** + * Direct SNI fallback for the CDN config GET: selection -> last-best IP -> + * builtin endpoints, first parseable and well-shaped response wins. + */ + private async fetchRemoteConfigViaSniFallback(): Promise { + try { + if (!isSupportIpTablePlatform() || !isSniSupported()) { + return null; + } + let proxyActive: boolean | null = null; + try { + proxyActive = await isProxyActiveForUrl(IP_TABLE_CDN_URL); + } catch { + return null; + } + if (proxyActive === true) { + // A user proxy owns routing; do not bypass it with SNI. + return null; + } + + const url = new URL(IP_TABLE_CDN_URL); + const rootDomain = url.hostname.split('.').slice(-2).join('.'); + const configWithRuntime = await this.getConfig(); + + const candidates: string[] = []; + const selectedIp = configWithRuntime.runtime?.selections?.[rootDomain]; + const lastBestIp = configWithRuntime.runtime?.lastBestIp?.[rootDomain]; + if (selectedIp) { + candidates.push(selectedIp); + } + if (lastBestIp) { + candidates.push(lastBestIp); + } + for (const endpoint of configWithRuntime.config.domains[rootDomain] + ?.endpoints ?? []) { + candidates.push(endpoint.ip); + } + const uniqueCandidates = [...new Set(candidates)].slice(0, 3); + if (uniqueCandidates.length === 0) { + return null; + } + + const headers = await getRequestHeaders(); + for (const ip of uniqueCandidates) { + try { + defaultLogger.ipTable.request.info({ + info: `[IpTable] CDN fetch fallback: trying SNI candidate for ${url.hostname}`, + }); + const response = await sniRequest({ + ip, + hostname: url.hostname, + path: `${url.pathname}${url.search}`, + headers, + method: 'GET', + body: null, + timeout: IP_TABLE_CDN_FETCH_TIMEOUT_MS, + }); + const body = response?.body ?? response?.data; + if (body) { + const parsed: unknown = + typeof body === 'string' ? JSON.parse(body) : body; + if (isValidIpTableRemoteConfigShape(parsed)) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] CDN fetch fallback succeeded via SNI, version: ${parsed.version}`, + }); + return parsed; + } + } + } catch (error) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] CDN fetch fallback candidate failed: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + }); + } + } + return null; + } catch (error) { + defaultLogger.ipTable.request.warn({ + info: `[IpTable] CDN fetch fallback error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + }); return null; } } From f5544cf98f9ce3dd7803d912bc463300823b32c7 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 11:14:12 +0800 Subject: [PATCH 15/22] fix: align failover kill switch semantics between main and bg runtimes --- packages/kit-bg/src/services/ServiceIpTable.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 134182029d84..17c7c5bf8768 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -1016,9 +1016,10 @@ class ServiceIpTable extends ServiceBase { private async isFailoverDisabledByDevSettings(): Promise { try { const devSettings = await devSettingsPersistAtom.get(); - return Boolean( - devSettings.enabled && devSettings.settings?.disableIpTableFailover, - ); + // Intentionally ignores devSettings.enabled to match the adapter-side + // check (and the existing disableIpTableInProd convention): main and + // bg runtimes must reach the same conclusion for the same settings. + return Boolean(devSettings.settings?.disableIpTableFailover); } catch { // Default to enabled when dev settings are unreadable. return false; From ea33f9881e1da8a884f7a7c47bc3892a084f3cfd Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 13:46:59 +0800 Subject: [PATCH 16/22] fix: order request outcomes by start time and dedupe sni failure reports --- .../kit-bg/src/services/ServiceIpTable.ts | 69 +++++++++--- .../request/helpers/ipTableAdapter.test.ts | 60 ++++++++++ .../src/request/helpers/ipTableAdapter.ts | 104 +++++++++++++----- 3 files changed, 189 insertions(+), 44 deletions(-) diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 17c7c5bf8768..24274d49d38b 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -65,6 +65,12 @@ interface IEndpointHealth { consecutiveFailures: number; /** Timestamp of last failure */ lastFailureTime: number; + /** + * Transport start time (Date.now) of the newest outcome applied to this + * endpoint. Success/failure reports race through async callbacks; an + * outcome whose request started before this point is stale and ignored. + */ + lastOutcomeAt: number; } /** @@ -141,6 +147,7 @@ class ServiceIpTable extends ServiceBase { failureCount: 0, consecutiveFailures: 0, lastFailureTime: 0, + lastOutcomeAt: 0, }, lastSpeedTestTime: 0, }; @@ -167,6 +174,7 @@ class ServiceIpTable extends ServiceBase { failureCount: 0, consecutiveFailures: 0, lastFailureTime: 0, + lastOutcomeAt: 0, }; stats.ipEndpoints.set(target, endpointHealth); } @@ -921,6 +929,7 @@ class ServiceIpTable extends ServiceBase { domain: string, requestType: 'ip' | 'domain', target: string, + startedAtMs?: number, ): Promise { // Check if IP Table is enabled if (!(await this.isIpTableEnabled())) { @@ -928,6 +937,7 @@ class ServiceIpTable extends ServiceBase { } const now = Date.now(); + const outcomeAt = startedAtMs ?? now; // Get or initialize domain health stats const stats = this.getDomainHealth(domain); @@ -935,6 +945,18 @@ class ServiceIpTable extends ServiceBase { // Get or initialize endpoint health const endpointHealth = this.getEndpointHealth(stats, requestType, target); + // Outcomes arrive through async callbacks with no ordering guarantee: + // ignore a failure whose request started before the newest outcome we + // already applied, so an old failure landing late cannot re-increment a + // counter that a newer success just cleared. + if (outcomeAt < endpointHealth.lastOutcomeAt) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Ignoring stale ${requestType} failure for ${domain} (${target}): request predates the newest applied outcome`, + }); + return; + } + endpointHealth.lastOutcomeAt = outcomeAt; + // Update failure statistics endpointHealth.failureCount += 1; endpointHealth.consecutiveFailures += 1; @@ -1065,27 +1087,42 @@ class ServiceIpTable extends ServiceBase { }); // Register request failure callback (handles both IP and domain failures) - setReportRequestFailureCallback(({ domain, requestType, target }) => { - void this.reportRequestFailure(domain, requestType, target); - }); + setReportRequestFailureCallback( + ({ domain, requestType, target, startedAtMs }) => { + void this.reportRequestFailure( + domain, + requestType, + target, + startedAtMs, + ); + }, + ); // Successes reset the consecutive-failure counters, so "consecutive" // means what it says: interleaved successes prevent sporadic errors // accumulated over hours from ever tripping the failover threshold. - setReportRequestSuccessCallback(({ domain, requestType, target }) => { - const stats = this.domainHealthMap.get(domain); - if (!stats) { - return; - } - if (requestType === 'domain') { - stats.domainDirect.consecutiveFailures = 0; - } else { - const ipHealth = stats.ipEndpoints.get(target); - if (ipHealth) { - ipHealth.consecutiveFailures = 0; + // Ordered by request start time: a slow success that started before the + // newest applied outcome must not clear failures that came after it. + setReportRequestSuccessCallback( + ({ domain, requestType, target, startedAtMs }) => { + const stats = this.domainHealthMap.get(domain); + if (!stats) { + return; } - } - }); + const endpointHealth = + requestType === 'domain' + ? stats.domainDirect + : stats.ipEndpoints.get(target); + if (!endpointHealth) { + return; + } + if (startedAtMs < endpointHealth.lastOutcomeAt) { + return; + } + endpointHealth.lastOutcomeAt = startedAtMs; + endpointHealth.consecutiveFailures = 0; + }, + ); // Try to refresh CDN config if needed const shouldRefresh = await this.shouldRefreshConfig(); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index fc5985a7c07f..e175db0bcadc 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -7,6 +7,7 @@ import requestHelper from '../requestHelper'; import { createIpTableAdapter, resetAdapterFailoverStatesForTesting, + setReportRequestFailureCallback, testIpSpeed, } from './ipTableAdapter'; import { isProxyActiveForUrl, isSniSupported, sniRequest } from './sniRequest'; @@ -378,6 +379,37 @@ describe('ipTableAdapter fail-open on domain network failures', () => { expect(mockedSniRequest).not.toHaveBeenCalled(); }); + test('an http error response resets the consecutive failure counter', async () => { + // 2 transport failures, then a 500 response, then 2 more transport + // failures: without the reset this would be 4 "consecutive" failures + // and the circuit would open despite proof the path works. + await failNTimes(2); + fallbackAdapter.mockImplementation(async () => { + throw httpError(500); + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ response: { status: 500 } }); + await failNTimes(2); + + fallbackAdapter.mockImplementation(async (config) => ({ + data: { fallback: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + })); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).resolves.toMatchObject({ data: { fallback: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + test('a success resets the consecutive failure counter', async () => { await failNTimes(2); fallbackAdapter.mockImplementation(async (config) => ({ @@ -779,4 +811,32 @@ describe('ipTableAdapter idempotency-gated fallback after SNI started', () => { }); expect(fallbackAdapter).not.toHaveBeenCalled(); }); + + test('one sni attempt reports at most one ip failure (null response + blocked fallback)', async () => { + // The null-response branch reports an ip failure, then throws for + // non-idempotent requests; the outer catch must not report the same + // failure a second time. + const failureSpy = jest.fn(); + setReportRequestFailureCallback(failureSpy); + try { + mockedSniRequest.mockResolvedValue(null); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ + message: expect.stringContaining('not idempotent'), + }); + expect(failureSpy).toHaveBeenCalledTimes(1); + expect(failureSpy).toHaveBeenCalledWith( + expect.objectContaining({ + requestType: 'ip', + target: '93.184.216.34', + startedAtMs: expect.any(Number), + }), + ); + } finally { + setReportRequestFailureCallback(() => undefined); + } + }); }); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 74bd4c1c3e4d..9edff26ccc8d 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -105,6 +105,12 @@ interface IRequestFailureParams { target: string; /** Error message */ error: string; + /** + * Date.now() captured when the request was handed to the transport. + * Outcomes race through async callbacks; consumers order them by request + * start time instead of processing time. + */ + startedAtMs: number; } let reportRequestFailureCallback: @@ -120,6 +126,8 @@ interface IRequestSuccessParams { domain: string; requestType: 'ip' | 'domain'; target: string; + /** Date.now() captured when the request was handed to the transport */ + startedAtMs: number; } let reportRequestSuccessCallback: @@ -875,23 +883,53 @@ export function createIpTableAdapter( domain: rootDomain, requestType: 'domain', target: hostname, + startedAtMs, }); } } return response; } catch (error) { + // A rejected promise carrying an HTTP response (validateStatus said + // no to a 4xx/5xx) still proves the transport path works: reset the + // counters exactly like a success before surfacing the error, so + // transport failures separated by healthy responses can never + // accumulate to the fail-open threshold. + const httpResponseReceived = Boolean( + error && + typeof error === 'object' && + 'response' in error && + (error as { response?: unknown }).response, + ); + if (rootDomain && hostname) { - // Track transport-level failures for adapter fail-open. The failing - // request itself is never replayed (no double-send of POSTs); only - // subsequent requests take the fail-open route. - await recordDomainRequestOutcome({ - rootDomain, - hostname, - ok: false, - startedAtMs, - error, - }); + if (httpResponseReceived) { + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: true, + startedAtMs, + }); + if (reportRequestSuccessCallback) { + reportRequestSuccessCallback({ + domain: rootDomain, + requestType: 'domain', + target: hostname, + startedAtMs, + }); + } + } else { + // Track transport-level failures for adapter fail-open. The + // failing request itself is never replayed (no double-send of + // POSTs); only subsequent requests take the fail-open route. + await recordDomainRequestOutcome({ + rootDomain, + hostname, + ok: false, + startedAtMs, + error, + }); + } } // Only report domain failures if this is NOT a fallback request, and @@ -913,6 +951,7 @@ export function createIpTableAdapter( requestType: 'domain', target: hostname, error: error instanceof Error ? error.message : String(error), + startedAtMs, }); } @@ -1158,6 +1197,26 @@ export function createIpTableAdapter( requestBody ? requestBody.substring(0, 200) : 'null', ); + const sniStartedAtMs = Date.now(); + // One SNI attempt must produce at most one ip-failure report: the + // null-response branch throws for non-idempotent requests and that + // throw lands in the same catch below, which would otherwise report the + // same failure a second time. + let ipFailureReported = false; + const reportIpFailureOnce = (errorMessage: string) => { + if (ipFailureReported || !reportRequestFailureCallback) { + return; + } + ipFailureReported = true; + reportRequestFailureCallback({ + domain: rootDomain, + requestType: 'ip', + target: selectedIp, + error: errorMessage, + startedAtMs: sniStartedAtMs, + }); + }; + try { const sniResponse = await sniRequest({ ip: selectedIp, @@ -1172,15 +1231,7 @@ export function createIpTableAdapter( // If SNI request fails, use original adapter if (!sniResponse) { debugLog('[IpTableAdapter] SNI request returned null, using fallback'); - // Report IP failure - if (reportRequestFailureCallback) { - reportRequestFailureCallback({ - domain: rootDomain, - requestType: 'ip', - target: selectedIp, - error: 'SNI response null', - }); - } + reportIpFailureOnce('SNI response null'); // A null response is ambiguous — the request may have reached the // server. Re-sending over the domain is only safe for idempotent // methods; anything else must surface the failure to the caller. @@ -1215,6 +1266,7 @@ export function createIpTableAdapter( domain: rootDomain, requestType: 'ip', target: selectedIp, + startedAtMs: sniStartedAtMs, }); } @@ -1267,15 +1319,11 @@ export function createIpTableAdapter( throw error; } - // Report IP failure if callback is registered - if (reportRequestFailureCallback) { - reportRequestFailureCallback({ - domain: rootDomain, - requestType: 'ip', - target: selectedIp, - error: error instanceof Error ? error.message : String(error), - }); - } + // Report IP failure if callback is registered (at most once per SNI + // attempt — the null-response branch may already have reported). + reportIpFailureOnce( + error instanceof Error ? error.message : String(error), + ); // Once the request was handed to the SNI transport it may have been // written to the wire (e.g. SNI_TIMEOUT after send). Re-sending over From 28a5f93a8669b75624365a026db1b1f8faeaff87 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 14:53:27 +0800 Subject: [PATCH 17/22] fix: unify outcome ordering in a shared start-time ledger for adapter and service --- .../kit-bg/src/services/ServiceIpTable.ts | 48 ++++++------- .../request/helpers/ipTableAdapter.test.ts | 70 +++++++++++++++++++ .../src/request/helpers/ipTableAdapter.ts | 64 ++++++++++------- .../helpers/ipTableOutcomeLedger.test.ts | 51 ++++++++++++++ .../request/helpers/ipTableOutcomeLedger.ts | 38 ++++++++++ 5 files changed, 219 insertions(+), 52 deletions(-) create mode 100644 packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts create mode 100644 packages/shared/src/request/helpers/ipTableOutcomeLedger.ts diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 24274d49d38b..14ac8cca4e0a 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -32,6 +32,7 @@ import { testIpSpeed, } from '@onekeyhq/shared/src/request/helpers/ipTableAdapter'; import { decideEndpoint } from '@onekeyhq/shared/src/request/helpers/ipTableEndpointDecision'; +import { applyOutcome } from '@onekeyhq/shared/src/request/helpers/ipTableOutcomeLedger'; import { isProxyActiveForUrl, isSniSupported, @@ -946,20 +947,21 @@ class ServiceIpTable extends ServiceBase { const endpointHealth = this.getEndpointHealth(stats, requestType, target); // Outcomes arrive through async callbacks with no ordering guarantee: - // ignore a failure whose request started before the newest outcome we - // already applied, so an old failure landing late cannot re-increment a - // counter that a newer success just cleared. - if (outcomeAt < endpointHealth.lastOutcomeAt) { + // the shared ledger orders them by transport start time, so a failure + // whose request predates the newest applied outcome (e.g. a success + // that landed first) never re-increments the counter. + if ( + applyOutcome(endpointHealth, { ok: false, startedAtMs: outcomeAt }) === + 'stale' + ) { defaultLogger.ipTable.request.info({ info: `[IpTable] Ignoring stale ${requestType} failure for ${domain} (${target}): request predates the newest applied outcome`, }); return; } - endpointHealth.lastOutcomeAt = outcomeAt; - // Update failure statistics + // Update failure statistics (consecutiveFailures is owned by the ledger) endpointHealth.failureCount += 1; - endpointHealth.consecutiveFailures += 1; endpointHealth.lastFailureTime = now; defaultLogger.ipTable.request.warn({ @@ -1099,28 +1101,20 @@ class ServiceIpTable extends ServiceBase { ); // Successes reset the consecutive-failure counters, so "consecutive" - // means what it says: interleaved successes prevent sporadic errors - // accumulated over hours from ever tripping the failover threshold. - // Ordered by request start time: a slow success that started before the - // newest applied outcome must not clear failures that came after it. + // means what it says. The success path MUST create the ledger entry when + // none exists yet: a later-started success completing before an + // early-started slow failure has to leave its ordering mark, otherwise + // the old failure would create the entry afterwards and count against a + // link that already proved healthy. setReportRequestSuccessCallback( ({ domain, requestType, target, startedAtMs }) => { - const stats = this.domainHealthMap.get(domain); - if (!stats) { - return; - } - const endpointHealth = - requestType === 'domain' - ? stats.domainDirect - : stats.ipEndpoints.get(target); - if (!endpointHealth) { - return; - } - if (startedAtMs < endpointHealth.lastOutcomeAt) { - return; - } - endpointHealth.lastOutcomeAt = startedAtMs; - endpointHealth.consecutiveFailures = 0; + const stats = this.getDomainHealth(domain); + const endpointHealth = this.getEndpointHealth( + stats, + requestType, + target, + ); + applyOutcome(endpointHealth, { ok: true, startedAtMs }); }, ); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index e175db0bcadc..21e188393476 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -621,6 +621,76 @@ describe('ipTableAdapter fail-open on domain network failures', () => { expect(lastSniCall.hostname).toBe('wallet.onekeycn.com'); }); + test('a late failure from a request started before a newer success does not count', async () => { + // Request A starts first and fails slowly; request B starts later and + // succeeds before A's failure lands. B's success must leave its ledger + // mark even though no failure entry existed yet, making A's late + // failure stale — otherwise a recovered hostname would accumulate + // toward fail-open. + let releaseSlowFail: (() => void) | undefined; + const slowFailGate = new Promise((resolve) => { + releaseSlowFail = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('slow-fail')) { + await slowFailGate; + throw networkError(); + } + if (config.url?.includes('ok-probe')) { + return { + data: { probe: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + const slowFailing = createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/slow-fail'), + ); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // Later-started success completes first. + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + + // Now the early-started failure lands: it must be ignored as stale. + releaseSlowFail?.(); + await expect(slowFailing).rejects.toMatchObject({ code: 'ECONNABORTED' }); + + // Two fresh failures: total applied failures is 2, not 3. + for (let i = 0; i < 2; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // The circuit must still be closed: the next request goes via domain. + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + test('a late success from a request started before activation does not close the circuit', async () => { // One shared implementation routed by URL marker so the in-flight stale // request is unaffected when the failing behavior is exercised. diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 9edff26ccc8d..24e0b50beade 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -12,10 +12,12 @@ import { import { getRequestHeaders } from '../Interceptor'; import requestHelper from '../requestHelper'; +import { applyOutcome, createOutcomeLedgerEntry } from './ipTableOutcomeLedger'; import { isSniFailClosedError } from './sniFailClosedError'; import { redactIpLiterals, safeSniLogValue } from './sniLogRedaction'; import { isProxyActiveForUrl, isSniSupported, sniRequest } from './sniRequest'; +import type { IIpTableOutcomeLedgerEntry } from './ipTableOutcomeLedger'; import type { AxiosAdapter, AxiosRequestConfig, @@ -233,14 +235,14 @@ const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { // restricted to idempotent methods or provably-unsent errors; see // canFallbackAfterSniStarted. -interface IHostFailureRecord { - consecutive: number; - lastFailureAt: number; -} - interface IAdapterFailoverState { - /** Consecutive transport failures tracked per hostname, not per root domain */ - hostFailures: Map; + /** + * Per-hostname outcome ledger (consecutive transport failures + newest + * applied outcome ordered by transport start time). Entries persist across + * successes so a late-arriving old failure can always be recognized as + * stale — deleting them would lose the ordering mark. + */ + hostFailures: Map; failOpenUntil: number; activatedAt: number; /** Hostnames whose failures opened the circuit; only they may close it early */ @@ -249,6 +251,18 @@ interface IAdapterFailoverState { fallbackIpIndex: number; } +function getHostOutcomeLedger( + state: IAdapterFailoverState, + hostname: string, +): IIpTableOutcomeLedgerEntry { + let entry = state.hostFailures.get(hostname); + if (!entry) { + entry = createOutcomeLedgerEntry(); + state.hostFailures.set(hostname, entry); + } + return entry; +} + const adapterFailoverStates = new Map(); /** Test-only helper: clears fail-open state and the selection memo cache. */ @@ -360,16 +374,20 @@ async function recordDomainRequestOutcome(options: { const lookupDomain = await resolveFailoverLookupDomain(rootDomain); const state = getFailoverState(lookupDomain); + // Both success and failure go through the same start-time-ordered ledger: + // a success records its order even on a fresh entry (so an older failure + // arriving later is recognized as stale), and a failure whose request + // started before the newest applied outcome never counts. + const ledger = getHostOutcomeLedger(state, hostname); + if (ok) { - const hostFailure = state.hostFailures.get(hostname); - // A success only proves the path that was exercised: it resets its own - // hostname's counter, and it may close the circuit only when it comes - // from a hostname that opened it AND started after activation (a - // long-in-flight request resolving late says nothing about recovery). - if (hostFailure && startedAtMs >= hostFailure.lastFailureAt) { - state.hostFailures.delete(hostname); - } + const applied = applyOutcome(ledger, { ok: true, startedAtMs }); + // A success only proves the path that was exercised: it may close the + // circuit only when it comes from a hostname that opened it AND started + // after activation (a long-in-flight request resolving late says + // nothing about recovery). if ( + applied === 'applied' && state.failOpenUntil > Date.now() && state.activatedHostnames.has(hostname) && startedAtMs >= state.activatedAt @@ -386,16 +404,12 @@ async function recordDomainRequestOutcome(options: { return; } - const now = Date.now(); - const hostFailure = state.hostFailures.get(hostname) ?? { - consecutive: 0, - lastFailureAt: 0, - }; - hostFailure.consecutive += 1; - hostFailure.lastFailureAt = now; - state.hostFailures.set(hostname, hostFailure); + if (applyOutcome(ledger, { ok: false, startedAtMs }) === 'stale') { + return; + } - if (hostFailure.consecutive >= IP_TABLE_DOMAIN_FAILOVER_THRESHOLD) { + if (ledger.consecutiveFailures >= IP_TABLE_DOMAIN_FAILOVER_THRESHOLD) { + const now = Date.now(); state.activatedHostnames.add(hostname); if (state.failOpenUntil <= now) { state.failOpenUntil = now + IP_TABLE_ADAPTER_FAILOVER_TTL_MS; @@ -405,7 +419,7 @@ async function recordDomainRequestOutcome(options: { logIpTableEvent('warn', 'adapter_failover_activated', { lookupDomain, hostname, - consecutiveNetworkFailures: hostFailure.consecutive, + consecutiveNetworkFailures: ledger.consecutiveFailures, ttlMs: IP_TABLE_ADAPTER_FAILOVER_TTL_MS, }); defaultLogger.ipTable.metrics.adapterFailover({ diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts new file mode 100644 index 000000000000..aa4cb24004c9 --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts @@ -0,0 +1,51 @@ +import { applyOutcome, createOutcomeLedgerEntry } from './ipTableOutcomeLedger'; + +describe('ipTableOutcomeLedger', () => { + it('regression: a newer success applied first blocks an older failure arriving late', () => { + // Request A starts at t=1 and fails slowly; request B starts at t=5 and + // succeeds first. B must leave a ledger mark even though the entry was + // fresh, so A's late failure is rejected as stale. + const entry = createOutcomeLedgerEntry(); + expect(applyOutcome(entry, { ok: true, startedAtMs: 5 })).toBe('applied'); + expect(applyOutcome(entry, { ok: false, startedAtMs: 1 })).toBe('stale'); + expect(entry.consecutiveFailures).toBe(0); + expect(entry.lastOutcomeAt).toBe(5); + }); + + it('old failure processed first is overridden by a newer success', () => { + const entry = createOutcomeLedgerEntry(); + expect(applyOutcome(entry, { ok: false, startedAtMs: 1 })).toBe('applied'); + expect(entry.consecutiveFailures).toBe(1); + expect(applyOutcome(entry, { ok: true, startedAtMs: 5 })).toBe('applied'); + expect(entry.consecutiveFailures).toBe(0); + }); + + it('both interleavings converge to the same final state', () => { + const a = createOutcomeLedgerEntry(); + applyOutcome(a, { ok: true, startedAtMs: 5 }); + applyOutcome(a, { ok: false, startedAtMs: 1 }); + + const b = createOutcomeLedgerEntry(); + applyOutcome(b, { ok: false, startedAtMs: 1 }); + applyOutcome(b, { ok: true, startedAtMs: 5 }); + + expect(a).toEqual(b); + }); + + it('consecutive failures accumulate only for non-stale outcomes', () => { + const entry = createOutcomeLedgerEntry(); + applyOutcome(entry, { ok: false, startedAtMs: 10 }); + applyOutcome(entry, { ok: false, startedAtMs: 11 }); + // stale failure from before the first one + applyOutcome(entry, { ok: false, startedAtMs: 5 }); + expect(entry.consecutiveFailures).toBe(2); + expect(entry.lastOutcomeAt).toBe(11); + }); + + it('equal start times apply in processing order (documented tie semantics)', () => { + const entry = createOutcomeLedgerEntry(); + applyOutcome(entry, { ok: true, startedAtMs: 5 }); + expect(applyOutcome(entry, { ok: false, startedAtMs: 5 })).toBe('applied'); + expect(entry.consecutiveFailures).toBe(1); + }); +}); diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts new file mode 100644 index 000000000000..845263054fc7 --- /dev/null +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts @@ -0,0 +1,38 @@ +/** + * Outcome ledger shared by the adapter's per-hostname fail-open tracking and + * the service's per-endpoint health stats. + * + * Request outcomes arrive through async callbacks with no ordering guarantee: + * an early-started request can fail slowly while a later-started request + * succeeds first. Ordering by transport start time (`startedAtMs`) — never by + * completion/processing time — makes every interleaving converge to the same + * state: an outcome whose request started before the newest applied outcome + * is stale and rejected, and both success and failure record their order even + * when the entry was previously empty. + */ +export interface IIpTableOutcomeLedgerEntry { + /** Consecutive failure count; reset to 0 by an applied success */ + consecutiveFailures: number; + /** Transport start time (Date.now) of the newest applied outcome */ + lastOutcomeAt: number; +} + +export function createOutcomeLedgerEntry(): IIpTableOutcomeLedgerEntry { + return { consecutiveFailures: 0, lastOutcomeAt: 0 }; +} + +export function applyOutcome( + entry: IIpTableOutcomeLedgerEntry, + outcome: { ok: boolean; startedAtMs: number }, +): 'applied' | 'stale' { + if (outcome.startedAtMs < entry.lastOutcomeAt) { + return 'stale'; + } + entry.lastOutcomeAt = outcome.startedAtMs; + if (outcome.ok) { + entry.consecutiveFailures = 0; + } else { + entry.consecutiveFailures += 1; + } + return 'applied'; +} From 26e1f8753b0915f4e3296390c0928b05bbb2c51a Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 16:01:00 +0800 Subject: [PATCH 18/22] fix: tighten sni pre-write allowlist and preserve failover watermarks on recovery --- .../request/helpers/ipTableAdapter.test.ts | 97 +++++++++++++++++++ .../src/request/helpers/ipTableAdapter.ts | 15 ++- 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index 21e188393476..7c55a6e1d762 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -691,6 +691,86 @@ describe('ipTableAdapter fail-open on domain network failures', () => { expect(mockedSniRequest).not.toHaveBeenCalled(); }); + test('after recovery closes the circuit, old failures landing later stay stale', async () => { + // Three slow requests start BEFORE anything goes wrong. The circuit then + // opens, recovers (which must preserve the per-hostname watermark), and + // only afterwards do the three old failures land: they must all be + // stale — if deactivation had cleared the ledger they would re-open the + // circuit on a link that already proved healthy. + let releaseOldFailures: (() => void) | undefined; + const oldFailureGate = new Promise((resolve) => { + releaseOldFailures = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('old-slow-fail')) { + await oldFailureGate; + throw networkError(); + } + if (config.url?.includes('ok-probe')) { + return { + data: { probe: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + const oldSlowFailures = [1, 2, 3].map((i) => + createIpTableAdapter({})( + buildConfig(`https://wallet.onekeycn.com/wallet/v1/old-slow-fail-${i}`), + ), + ); + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + + // Open the circuit with 3 fresh transport failures. + for (let i = 0; i < 3; i += 1) { + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/health'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // Recover: circuit is open, so the probe goes SNI first; an ambiguous + // timeout on a GET falls back to the domain, and that domain success + // (activated hostname, started after activation) closes the circuit. + mockedSniRequest.mockRejectedValueOnce( + Object.assign(new Error('sni timeout'), { code: 'SNI_TIMEOUT' }), + ); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + + // Old failures land only now — after recovery. + releaseOldFailures?.(); + for (const pending of oldSlowFailures) { + await expect(pending).rejects.toMatchObject({ code: 'ECONNABORTED' }); + } + + // Circuit must still be closed: next request goes via domain, not SNI. + mockedSniRequest.mockClear(); + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"ok":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/ok-probe'), + ), + ).resolves.toMatchObject({ data: { probe: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + test('a late success from a request started before activation does not close the circuit', async () => { // One shared implementation routed by URL marker so the in-flight stale // request is unaffected when the failing behavior is exercised. @@ -870,6 +950,23 @@ describe('ipTableAdapter idempotency-gated fallback after SNI started', () => { expect(fallbackAdapter).toHaveBeenCalledTimes(1); }); + test('POST does NOT fall back on SNI_NETWORK_UNREACHABLE (iOS maps connection-lost to it)', async () => { + // NSURLErrorNetworkConnectionLost — which can fire after the body was + // sent — maps to SNI_NETWORK_UNREACHABLE on iOS, so this code cannot + // prove the request was never written. + mockedSniRequest.mockRejectedValue( + Object.assign(new Error('network unreachable'), { + code: 'SNI_NETWORK_UNREACHABLE', + }), + ); + await expect( + createIpTableAdapter({})( + buildMethodConfig('https://api.example.com/v1', 'post'), + ), + ).rejects.toMatchObject({ code: 'SNI_NETWORK_UNREACHABLE' }); + expect(fallbackAdapter).not.toHaveBeenCalled(); + }); + test('POST does NOT fall back when SNI returns a null response', async () => { mockedSniRequest.mockResolvedValue(null); await expect( diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 24e0b50beade..1a01e8379c70 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -150,10 +150,15 @@ export function setReportRequestSuccessCallback( */ const IDEMPOTENT_HTTP_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']); +// SNI_NETWORK_UNREACHABLE is deliberately NOT in this set: on iOS the native +// module maps NSURLErrorNetworkConnectionLost — which can fire after the body +// was fully sent and the server executed the request — to that code +// (SniConnectClient.swift maps NotConnectedToInternet and +// NetworkConnectionLost to the same .networkUnreachable case), so it cannot +// prove the request was never written. const SNI_PRE_WRITE_ERROR_CODES = new Set([ 'SNI_CONNECTION_REFUSED', 'SNI_DNS_FAILED', - 'SNI_NETWORK_UNREACHABLE', 'SNI_INVALID_URL', ]); @@ -348,7 +353,13 @@ function deactivateFailover(lookupDomain: string, cause: string): void { action: 'deactivated', }); } - state.hostFailures.clear(); + // Reset counters but PRESERVE each hostname's lastOutcomeAt watermark: + // clearing entries would let failures from requests started before the + // recovery re-apply on fresh entries and reopen a circuit that the link + // already proved healthy. + state.hostFailures.forEach((entry) => { + entry.consecutiveFailures = 0; + }); state.failOpenUntil = 0; state.activatedAt = 0; state.activatedHostnames.clear(); From 153274f971ce3e38f6c7f27462d79f757c5f8a2a Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 17 Jul 2026 16:01:00 +0800 Subject: [PATCH 19/22] fix: apply signed config verbatim with runtime pruning and per-host direct health --- .../simple/entity/SimpleDbEntityIpTable.ts | 62 +-- .../kit-bg/src/services/ServiceIpTable.ts | 84 ++-- .../shared/src/utils/ipTableUtils.test.ts | 401 +++--------------- packages/shared/src/utils/ipTableUtils.ts | 67 +-- 4 files changed, 195 insertions(+), 419 deletions(-) diff --git a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts index a48575e577a9..2821e1ca6def 100644 --- a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts +++ b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts @@ -5,6 +5,7 @@ import type { IIpTableRemoteConfig, IIpTableRuntime, } from '@onekeyhq/shared/src/request/types/ipTable'; +import { pruneIpTableRuntimeSelections } from '@onekeyhq/shared/src/utils/ipTableUtils'; import { SimpleDbEntityBase } from '../base/SimpleDbEntityBase'; @@ -64,31 +65,44 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase { - await this.setRawData((data) => ({ - ...data, - config, - runtime: { - ...(data?.runtime ?? { - enabled: true, - lastUpdated: Date.now(), - lastRegionCheck: 0, - selections: {}, - }), + await this.setRawData((data) => { + const runtime = data?.runtime ?? { + enabled: true, lastUpdated: Date.now(), - // Provenance of the last successfully verified config; consumed by - // rollback protection and diagnostics. - ...(verifiedMeta - ? { - lastVerified: { - at: Date.now(), - version: config.version, - generatedAt: config.generated_at, - payloadHash: verifiedMeta.payloadHash, - }, - } - : {}), - }, - })); + lastRegionCheck: 0, + selections: {}, + }; + // The stored config is the signed envelope verbatim, so runtime state + // pointing at endpoints the new config no longer endorses (revoked or + // rotated-out IPs) must be dropped alongside it. + const pruned = pruneIpTableRuntimeSelections({ + config, + selections: runtime.selections, + lastBestIp: runtime.lastBestIp, + }); + return { + ...data, + config, + runtime: { + ...runtime, + selections: pruned.selections, + lastBestIp: pruned.lastBestIp, + lastUpdated: Date.now(), + // Provenance of the last successfully verified config; consumed by + // rollback protection and diagnostics. + ...(verifiedMeta + ? { + lastVerified: { + at: Date.now(), + version: config.version, + generatedAt: config.generated_at, + payloadHash: verifiedMeta.payloadHash, + }, + } + : {}), + }, + }; + }); } /** diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 14ac8cca4e0a..b370ebffab8a 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -48,7 +48,6 @@ import { isIpTableConfigRegression, isSupportIpTablePlatform, isValidIpTableRemoteConfigShape, - mergeIpTableConfigs, verifyIpTableConfigSignatureDetailed, } from '@onekeyhq/shared/src/utils/ipTableUtils'; @@ -81,8 +80,14 @@ interface IEndpointHealth { interface IDomainHealthStats { /** Health stats for each IP endpoint */ ipEndpoints: Map; - /** Health stats for direct domain connection */ - domainDirect: IEndpointHealth; + /** + * Direct-domain health tracked per hostname (wallet./utility./...): a + * success on one hostname must not clear another hostname's failures, and + * different hostnames' failures must not jointly reach a threshold. Root + * domain routing decisions aggregate over these explicitly. Mirrors the + * adapter-side per-hostname ledger so main and bg runtimes agree. + */ + directHosts: Map; /** Timestamp of last speed test */ lastSpeedTestTime: number; } @@ -144,12 +149,7 @@ class ServiceIpTable extends ServiceBase { if (!stats) { stats = { ipEndpoints: new Map(), - domainDirect: { - failureCount: 0, - consecutiveFailures: 0, - lastFailureTime: 0, - lastOutcomeAt: 0, - }, + directHosts: new Map(), lastSpeedTestTime: 0, }; this.domainHealthMap.set(domain, stats); @@ -165,11 +165,12 @@ class ServiceIpTable extends ServiceBase { requestType: 'ip' | 'domain', target: string, ): IEndpointHealth { - if (requestType === 'domain') { - return stats.domainDirect; - } - - let endpointHealth = stats.ipEndpoints.get(target); + // For domain requests, target is the hostname; for ip requests it is the + // ip. Both are tracked per target so outcomes of one never contaminate + // another's counters. + const bucket = + requestType === 'domain' ? stats.directHosts : stats.ipEndpoints; + let endpointHealth = bucket.get(target); if (!endpointHealth) { endpointHealth = { failureCount: 0, @@ -177,11 +178,20 @@ class ServiceIpTable extends ServiceBase { lastFailureTime: 0, lastOutcomeAt: 0, }; - stats.ipEndpoints.set(target, endpointHealth); + bucket.set(target, endpointHealth); } return endpointHealth; } + /** Highest consecutive direct-domain failure count across hostnames */ + private getMaxDirectConsecutiveFailures(stats: IDomainHealthStats): number { + let max = 0; + stats.directHosts.forEach((health) => { + max = Math.max(max, health.consecutiveFailures); + }); + return max; + } + /** * Check if speed test should be triggered based on health stats * Returns true only when current endpoint is failing AND cooldown period has passed @@ -208,8 +218,11 @@ class ServiceIpTable extends ServiceBase { // never triggering (the old `return false` deadlocked selection forever // when the initial speed test had not completed). if (currentSelection === undefined || currentSelection === '') { + // Aggregate across hostnames: any single hostname consistently failing + // is enough to justify a probing round for the root domain. return ( - stats.domainDirect.consecutiveFailures >= IP_TABLE_SNI_FAILURE_THRESHOLD + this.getMaxDirectConsecutiveFailures(stats) >= + IP_TABLE_SNI_FAILURE_THRESHOLD ); } @@ -471,7 +484,7 @@ class ServiceIpTable extends ServiceBase { } @backgroundMethod() - async fetchAndMergeRemoteConfig(): Promise { + async fetchAndApplyRemoteConfig(): Promise { try { const remoteConfig = await this.fetchRemoteConfig(); @@ -525,15 +538,19 @@ class ServiceIpTable extends ServiceBase { return false; } - const mergedConfig = mergeIpTableConfigs(localConfig, remoteConfig); - + // Persist the signed remote envelope VERBATIM. Merging local endpoints + // in would make the effective config diverge from what the signature + // (and the persisted payloadHash) covers, and would make endpoints + // irrevocable: a compromised or rotated-out IP removed from the CDN + // config must actually disappear, including any runtime selection or + // last-best IP that still points at it (pruned inside saveConfig). defaultLogger.ipTable.request.info({ - info: `[IpTable] Merged config has ${ - Object.keys(mergedConfig.domains).length + info: `[IpTable] Applying signed remote config with ${ + Object.keys(remoteConfig.domains).length } domains`, }); - await this.backgroundApi.simpleDb.ipTable.saveConfig(mergedConfig, { + await this.backgroundApi.simpleDb.ipTable.saveConfig(remoteConfig, { payloadHash: computeIpTableConfigHash(remoteConfig), }); @@ -543,7 +560,7 @@ class ServiceIpTable extends ServiceBase { return true; } catch (error) { defaultLogger.ipTable.request.error({ - info: `[IpTable] Error in fetchAndMergeRemoteConfig: ${ + info: `[IpTable] Error in fetchAndApplyRemoteConfig: ${ error instanceof Error ? error.message : 'Unknown error' }`, }); @@ -684,6 +701,21 @@ class ServiceIpTable extends ServiceBase { domain: string, opts?: { trigger?: 'domain_failure' | 'ip_failure' | 'periodic' }, ): Promise { + // The kill switch must disable EVERY failover behavior, including the + // reachability-first override that `domain_failure` grants inside + // decideEndpoint (which ignores the 30% threshold). Downgrading the + // trigger here covers the decision override, the stale-write authority + // and the coalescing escalation in one place. + if ( + opts?.trigger === 'domain_failure' && + (await this.isFailoverDisabledByDevSettings()) + ) { + defaultLogger.ipTable.request.info({ + info: `[IpTable] Failover disabled by dev settings: downgrading domain_failure trigger to periodic for ${domain}`, + }); + // eslint-disable-next-line no-param-reassign + opts = { trigger: 'periodic' }; + } const inflight = this.speedTestInFlight.get(domain); if (inflight) { // Do not let a stronger trigger be swallowed by a weaker in-flight @@ -1026,7 +1058,9 @@ class ServiceIpTable extends ServiceBase { stats.lastSpeedTestTime = now; // Reset all consecutive failure counters for this domain (they'll be recounted after speed test) - stats.domainDirect.consecutiveFailures = 0; + stats.directHosts.forEach((health) => { + health.consecutiveFailures = 0; + }); stats.ipEndpoints.forEach((health) => { health.consecutiveFailures = 0; }); @@ -1126,7 +1160,7 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.info({ info: '[IpTable] CDN config refresh needed, fetching remote config', }); - const configUpdated = await this.fetchAndMergeRemoteConfig(); + const configUpdated = await this.fetchAndApplyRemoteConfig(); needSpeedTest = true; if (configUpdated) { diff --git a/packages/shared/src/utils/ipTableUtils.test.ts b/packages/shared/src/utils/ipTableUtils.test.ts index 45b04a1ec8b5..4313044e45ef 100644 --- a/packages/shared/src/utils/ipTableUtils.test.ts +++ b/packages/shared/src/utils/ipTableUtils.test.ts @@ -8,7 +8,7 @@ import { computeIpTableConfigHash, isIpTableConfigRegression, isValidIpTableRemoteConfigShape, - mergeIpTableConfigs, + pruneIpTableRuntimeSelections, verifyIpTableConfigSignature, verifyIpTableConfigSignatureDetailed, } from './ipTableUtils'; @@ -375,353 +375,70 @@ describe('computeIpTableConfigHash', () => { }); }); -describe('mergeIpTableConfigs', () => { - test('merges new domains from remote config', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'local.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'remote.com': { - endpoints: [ - { ip: '2.2.2.2', provider: 'remote', region: 'CN', weight: 50 }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['local.com']).toBeDefined(); - expect(merged.domains['remote.com']).toBeDefined(); - expect(merged.domains['local.com'].endpoints).toHaveLength(1); - expect(merged.domains['remote.com'].endpoints).toHaveLength(1); - }); - - test('merges new endpoints for existing domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'example.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'example.com': { - endpoints: [ - { ip: '2.2.2.2', provider: 'remote', region: 'CN', weight: 50 }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['example.com'].endpoints).toHaveLength(2); - expect(merged.domains['example.com'].endpoints[0].ip).toBe('1.1.1.1'); - expect(merged.domains['example.com'].endpoints[1].ip).toBe('2.2.2.2'); - }); - - test('deduplicates endpoints with same IP', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'example.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - { ip: '2.2.2.2', provider: 'local2', region: 'GLOBAL', weight: 80 }, - ], - }, +describe('pruneIpTableRuntimeSelections', () => { + const config: IIpTableRemoteConfig = { + version: 2, + ttl_sec: 86_400, + generated_at: '2026-07-17T00:00:00.000Z', + signature: '0xnew', + domains: { + 'onekeycn.com': { + endpoints: [ + { ip: '1.1.1.1', provider: 'a', region: 'GLOBAL', weight: 100 }, + { ip: '2.2.2.2', provider: 'b', region: 'CN', weight: 100 }, + ], }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'example.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'remote', region: 'CN', weight: 50 }, // Duplicate IP - { ip: '3.3.3.3', provider: 'remote2', region: 'CN', weight: 60 }, // New IP - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - // Should have 3 endpoints: 1.1.1.1 (from local), 2.2.2.2 (from local), 3.3.3.3 (from remote) - expect(merged.domains['example.com'].endpoints).toHaveLength(3); - - const ips = merged.domains['example.com'].endpoints.map((ep) => ep.ip); - expect(ips).toEqual(['1.1.1.1', '2.2.2.2', '3.3.3.3']); - }); - - test('uses remote config metadata (version, ttl, generated_at, signature)', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'local.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'remote.com': { - endpoints: [ - { ip: '2.2.2.2', provider: 'remote', region: 'CN', weight: 50 }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.version).toBe(2); - expect(merged.ttl_sec).toBe(43_200); - expect(merged.generated_at).toBe('2025-11-06T12:00:00.000Z'); - expect(merged.signature).toBe('0xremote'); - }); - - test('preserves all local endpoints when merging', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'example.com': { - endpoints: [ - { - ip: '1.1.1.1', - provider: 'provider1', - region: 'GLOBAL', - weight: 100, - }, - { - ip: '2.2.2.2', - provider: 'provider2', - region: 'GLOBAL', - weight: 90, - }, - { ip: '3.3.3.3', provider: 'provider3', region: 'CN', weight: 80 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'example.com': { - endpoints: [ - { - ip: '4.4.4.4', - provider: 'provider4', - region: 'GLOBAL', - weight: 70, - }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['example.com'].endpoints).toHaveLength(4); - - // Verify all local endpoints are preserved - const ips = merged.domains['example.com'].endpoints.map((ep) => ep.ip); - expect(ips).toContain('1.1.1.1'); - expect(ips).toContain('2.2.2.2'); - expect(ips).toContain('3.3.3.3'); - expect(ips).toContain('4.4.4.4'); + }, + }; + + test('keeps selections and last-best ips the config still endorses', () => { + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'onekeycn.com': '1.1.1.1' }, + lastBestIp: { 'onekeycn.com': '2.2.2.2' }, + }); + expect(result.selections).toEqual({ 'onekeycn.com': '1.1.1.1' }); + expect(result.lastBestIp).toEqual({ 'onekeycn.com': '2.2.2.2' }); + expect(result.prunedCount).toBe(0); + }); + + test('drops selections pointing at revoked endpoints', () => { + // 9.9.9.9 was removed from the signed config (revoked / rotated out): + // neither the selection nor the last-best ip may keep routing to it. + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'onekeycn.com': '9.9.9.9' }, + lastBestIp: { 'onekeycn.com': '9.9.9.9' }, + }); + expect(result.selections).toEqual({}); + expect(result.lastBestIp).toEqual({}); + expect(result.prunedCount).toBe(2); }); - test('handles empty local domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: {}, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'remote.com': { - endpoints: [ - { - ip: '1.1.1.1', - provider: 'remote', - region: 'GLOBAL', - weight: 100, - }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['remote.com']).toBeDefined(); - expect(merged.domains['remote.com'].endpoints).toHaveLength(1); + test('always keeps the explicit domain choice (empty string)', () => { + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'onekeycn.com': '' }, + }); + expect(result.selections).toEqual({ 'onekeycn.com': '' }); + expect(result.prunedCount).toBe(0); }); - test('handles empty remote domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'local.com': { - endpoints: [ - { ip: '1.1.1.1', provider: 'local', region: 'GLOBAL', weight: 100 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: {}, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(merged.domains['local.com']).toBeDefined(); - expect(merged.domains['local.com'].endpoints).toHaveLength(1); + test('drops state for domains the config no longer covers', () => { + const result = pruneIpTableRuntimeSelections({ + config, + selections: { 'gone.com': '1.1.1.1' }, + lastBestIp: { 'gone.com': '1.1.1.1' }, + }); + expect(result.selections).toEqual({}); + expect(result.lastBestIp).toEqual({}); + expect(result.prunedCount).toBe(2); }); - test('handles complex merge scenario with multiple domains', () => { - const localConfig: IIpTableRemoteConfig = { - version: 1, - ttl_sec: 86_400, - generated_at: '2025-11-06T08:00:00.000Z', - signature: '0xlocal', - domains: { - 'onekeycn.com': { - endpoints: [ - { - ip: '104.18.20.233', - provider: 'cloudflare', - region: 'GLOBAL', - weight: 100, - }, - { - ip: '216.19.4.106', - provider: 'volcengine', - region: 'CN', - weight: 100, - }, - ], - }, - 'localonly.com': { - endpoints: [ - { ip: '5.5.5.5', provider: 'local', region: 'GLOBAL', weight: 50 }, - ], - }, - }, - }; - - const remoteConfig: IIpTableRemoteConfig = { - version: 2, - ttl_sec: 43_200, - generated_at: '2025-11-06T12:00:00.000Z', - signature: '0xremote', - domains: { - 'onekeycn.com': { - endpoints: [ - { - ip: '104.18.20.233', - provider: 'cloudflare', - region: 'GLOBAL', - weight: 100, - }, // Duplicate - { ip: '6.6.6.6', provider: 'newcdn', region: 'CN', weight: 90 }, // New - ], - }, - 'remoteonly.com': { - endpoints: [ - { ip: '7.7.7.7', provider: 'remote', region: 'GLOBAL', weight: 80 }, - ], - }, - }, - }; - - const merged = mergeIpTableConfigs(localConfig, remoteConfig); - - expect(Object.keys(merged.domains)).toHaveLength(3); - - expect(merged.domains['onekeycn.com'].endpoints).toHaveLength(3); - const onekeyIps = merged.domains['onekeycn.com'].endpoints.map( - (ep) => ep.ip, - ); - expect(onekeyIps).toEqual(['104.18.20.233', '216.19.4.106', '6.6.6.6']); - - expect(merged.domains['localonly.com'].endpoints).toHaveLength(1); - expect(merged.domains['localonly.com'].endpoints[0].ip).toBe('5.5.5.5'); - - expect(merged.domains['remoteonly.com'].endpoints).toHaveLength(1); - expect(merged.domains['remoteonly.com'].endpoints[0].ip).toBe('7.7.7.7'); - - // Metadata should be from remote - expect(merged.version).toBe(2); - expect(merged.ttl_sec).toBe(43_200); - expect(merged.signature).toBe('0xremote'); + test('handles missing runtime maps', () => { + const result = pruneIpTableRuntimeSelections({ config }); + expect(result.selections).toEqual({}); + expect(result.lastBestIp).toEqual({}); + expect(result.prunedCount).toBe(0); }); }); diff --git a/packages/shared/src/utils/ipTableUtils.ts b/packages/shared/src/utils/ipTableUtils.ts index eee96efcbb6f..b15255cf9f7f 100644 --- a/packages/shared/src/utils/ipTableUtils.ts +++ b/packages/shared/src/utils/ipTableUtils.ts @@ -224,38 +224,49 @@ export function isIpTableConfigRegression(options: { return { regression: false }; } -export function mergeIpTableConfigs( - localConfig: IIpTableRemoteConfig, - remoteConfig: IIpTableRemoteConfig, -): IIpTableRemoteConfig { - const mergedDomains = { ...localConfig.domains }; - - for (const [domain, remoteDomainConfig] of Object.entries( - remoteConfig.domains, - )) { - if (mergedDomains[domain]) { - const localEndpoints = mergedDomains[domain].endpoints; - const remoteEndpoints = remoteDomainConfig.endpoints; - - const existingIps = new Set(localEndpoints.map((ep) => ep.ip)); +/** + * Drop runtime selections and last-best IPs that the current signed config + * no longer endorses. The persisted effective config is the signed remote + * envelope verbatim (never an additive merge), so a revoked or compromised + * endpoint removed from the CDN config must also stop being routable via + * previously persisted runtime state. + */ +export function pruneIpTableRuntimeSelections(options: { + config: IIpTableRemoteConfig; + selections?: Record; + lastBestIp?: Record; +}): { + selections: Record; + lastBestIp: Record; + prunedCount: number; +} { + const allowedIpsByDomain = new Map>(); + for (const [domain, domainConfig] of Object.entries(options.config.domains)) { + allowedIpsByDomain.set( + domain, + new Set(domainConfig.endpoints.map((endpoint) => endpoint.ip)), + ); + } - const newEndpoints = remoteEndpoints.filter( - (ep) => !existingIps.has(ep.ip), - ); + let prunedCount = 0; + const selections: Record = {}; + for (const [domain, ip] of Object.entries(options.selections ?? {})) { + // '' is the explicit "use the domain" choice and is always allowed. + if (ip === '' || allowedIpsByDomain.get(domain)?.has(ip)) { + selections[domain] = ip; + } else { + prunedCount += 1; + } + } - mergedDomains[domain] = { - endpoints: [...localEndpoints, ...newEndpoints], - }; + const lastBestIp: Record = {}; + for (const [domain, ip] of Object.entries(options.lastBestIp ?? {})) { + if (allowedIpsByDomain.get(domain)?.has(ip)) { + lastBestIp[domain] = ip; } else { - mergedDomains[domain] = remoteDomainConfig; + prunedCount += 1; } } - return { - version: remoteConfig.version, - ttl_sec: remoteConfig.ttl_sec, - generated_at: remoteConfig.generated_at, - signature: remoteConfig.signature, - domains: mergedDomains, - }; + return { selections, lastBestIp, prunedCount }; } From d1ce18dc9000246cfa6412051128781ab4ea78a9 Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 21 Jul 2026 16:32:54 +0800 Subject: [PATCH 20/22] fix: address ip table review feedback --- .../services/ServiceIpTable.review.test.ts | 177 ++++++++++++++++++ .../kit-bg/src/services/ServiceIpTable.ts | 92 +++++++-- .../request/helpers/ipTableAdapter.test.ts | 5 +- .../src/request/helpers/ipTableAdapter.ts | 71 +++---- .../helpers/ipTableOutcomeLedger.test.ts | 56 ++++-- .../request/helpers/ipTableOutcomeLedger.ts | 30 +-- 6 files changed, 347 insertions(+), 84 deletions(-) create mode 100644 packages/kit-bg/src/services/ServiceIpTable.review.test.ts diff --git a/packages/kit-bg/src/services/ServiceIpTable.review.test.ts b/packages/kit-bg/src/services/ServiceIpTable.review.test.ts new file mode 100644 index 000000000000..79f717040df4 --- /dev/null +++ b/packages/kit-bg/src/services/ServiceIpTable.review.test.ts @@ -0,0 +1,177 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access */ + +import type { IIpTableRemoteConfig } from '@onekeyhq/shared/src/request/types/ipTable'; + +import ServiceIpTable from './ServiceIpTable'; + +const mockAxiosGet = jest.fn(); + +jest.mock('@onekeyhq/shared/src/background/backgroundDecorators', () => ({ + backgroundClass: () => (target: unknown) => target, + backgroundMethod: + () => (_target: unknown, _key: string, descriptor: PropertyDescriptor) => + descriptor, +})); + +jest.mock('@onekeyhq/shared/src/logger/logger', () => ({ + defaultLogger: { + ipTable: { + request: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, + metrics: { + endpointSwitched: jest.fn(), + configVerifyFailed: jest.fn(), + }, + }, + }, +})); + +jest.mock('@onekeyhq/shared/src/request/helpers/ipTableAdapter', () => ({ + createAxiosWithIpTable: jest.fn(() => ({ get: mockAxiosGet })), + getSelectedIpForHost: Object.assign(jest.fn(), { clear: jest.fn() }), + setReportRequestFailureCallback: jest.fn(), + setReportRequestSuccessCallback: jest.fn(), + testDomainSpeed: jest.fn(), + testIpSpeed: jest.fn(), +})); + +jest.mock('@onekeyhq/shared/src/request/helpers/sniRequest', () => ({ + isProxyActiveForUrl: jest.fn(async () => false), + isSniSupported: jest.fn(() => true), + sniRequest: jest.fn(), +})); + +jest.mock('@onekeyhq/shared/src/request/Interceptor', () => ({ + getRequestHeaders: jest.fn(async () => ({})), +})); + +jest.mock('../states/jotai/atoms', () => ({ + devSettingsPersistAtom: { + get: jest.fn(async () => ({ enabled: false, settings: {} })), + }, +})); + +jest.mock('./ServiceBase', () => ({ + __esModule: true, + default: class ServiceBaseMock { + backgroundApi: any; + + constructor({ backgroundApi }: { backgroundApi: any }) { + this.backgroundApi = backgroundApi; + } + }, +})); + +const DOMAIN = 'onekeycn.com'; + +function buildConfig(ip: string, version = 1): IIpTableRemoteConfig { + return { + version, + ttl_sec: 300, + generated_at: `2026-07-${String(version).padStart(2, '0')}T00:00:00.000Z`, + signature: `signature-${version}`, + domains: { + [DOMAIN]: { + endpoints: [ + { + ip, + provider: 'test', + region: 'ALL', + weight: 1, + }, + ], + }, + }, + }; +} + +function createService() { + const ipTableDb = { + getConfig: jest.fn(), + updateLastBestIp: jest.fn(async () => undefined), + updateSelection: jest.fn(async () => undefined), + }; + const service = new ServiceIpTable({ + backgroundApi: { simpleDb: { ipTable: ipTableDb } }, + }); + return { service, ipTableDb }; +} + +describe('ServiceIpTable review regressions', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + ['empty', null], + ['invalid shape', 'captive portal'], + ])( + 'uses SNI fallback for an HTTP 200 %s CDN response', + async (_name, data) => { + const { service } = createService(); + const fallbackConfig = buildConfig('1.1.1.1'); + mockAxiosGet.mockResolvedValueOnce({ data }); + const fallbackSpy = jest + .spyOn(service as any, 'fetchRemoteConfigViaSniFallback') + .mockResolvedValue(fallbackConfig); + + await expect((service as any).fetchRemoteConfig()).resolves.toEqual( + fallbackConfig, + ); + expect(fallbackSpy).toHaveBeenCalledTimes(1); + }, + ); + + it('discards and queues a rerun when the signed config changes during a speed test', async () => { + const { service, ipTableDb } = createService(); + const originalConfig = buildConfig('1.1.1.1', 1); + // Keep the numeric version unchanged: the signed payload hash and the + // currently endorsed endpoint set must still invalidate the old round. + const replacementConfig = buildConfig('2.2.2.2', 1); + + jest.spyOn(service as any, 'isIpTableEnabled').mockResolvedValue(true); + jest + .spyOn(service as any, 'testMultipleTimes') + .mockResolvedValueOnce(100) + .mockResolvedValueOnce(20); + jest + .spyOn(service, 'getConfig') + .mockResolvedValueOnce({ config: originalConfig, runtime: undefined }) + .mockResolvedValueOnce({ config: replacementConfig, runtime: undefined }); + + await (service as any).selectBestEndpointForDomainInternal(DOMAIN, { + trigger: 'periodic', + }); + + expect(ipTableDb.updateLastBestIp).not.toHaveBeenCalled(); + expect(ipTableDb.updateSelection).not.toHaveBeenCalled(); + expect((service as any).pendingConfigChangeRerun.get(DOMAIN)).toBe( + 'periodic', + ); + }); + + it('starts the queued config-change rerun after the stale round settles', async () => { + const { service } = createService(); + const internalSpy = jest + .spyOn(service as any, 'selectBestEndpointForDomainInternal') + .mockImplementationOnce(async () => { + (service as any).pendingConfigChangeRerun.set(DOMAIN, 'ip_failure'); + }) + .mockResolvedValueOnce(undefined); + + await service.selectBestEndpointForDomain(DOMAIN, { + trigger: 'ip_failure', + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(internalSpy).toHaveBeenCalledTimes(2); + expect(internalSpy).toHaveBeenNthCalledWith(2, DOMAIN, { + trigger: 'ip_failure', + }); + }); +}); diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index b370ebffab8a..96406d2afecd 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -32,7 +32,10 @@ import { testIpSpeed, } from '@onekeyhq/shared/src/request/helpers/ipTableAdapter'; import { decideEndpoint } from '@onekeyhq/shared/src/request/helpers/ipTableEndpointDecision'; -import { applyOutcome } from '@onekeyhq/shared/src/request/helpers/ipTableOutcomeLedger'; +import { + applyOutcome, + nextIpTableRequestSequence, +} from '@onekeyhq/shared/src/request/helpers/ipTableOutcomeLedger'; import { isProxyActiveForUrl, isSniSupported, @@ -66,11 +69,11 @@ interface IEndpointHealth { /** Timestamp of last failure */ lastFailureTime: number; /** - * Transport start time (Date.now) of the newest outcome applied to this - * endpoint. Success/failure reports race through async callbacks; an - * outcome whose request started before this point is stale and ignored. + * Runtime-local request sequence of the newest outcome applied to this + * endpoint. Success/failure reports race through async callbacks; an older + * request sequence is stale and ignored even when Date.now() tied. */ - lastOutcomeAt: number; + lastOutcomeSequence: number; } /** @@ -176,7 +179,7 @@ class ServiceIpTable extends ServiceBase { failureCount: 0, consecutiveFailures: 0, lastFailureTime: 0, - lastOutcomeAt: 0, + lastOutcomeSequence: 0, }; bucket.set(target, endpointHealth); } @@ -350,7 +353,7 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.error({ info: '[IpTable] CDN returned empty config', }); - return null; + return this.fetchRemoteConfigViaSniFallback(); } // A misconfigured CDN edge can return HTML or a truncated body that @@ -360,7 +363,7 @@ class ServiceIpTable extends ServiceBase { defaultLogger.ipTable.request.error({ info: '[IpTable] Skipping CDN config update: invalid config shape (delivery-layer problem)', }); - return null; + return this.fetchRemoteConfigViaSniFallback(); } defaultLogger.ipTable.request.info({ @@ -696,6 +699,14 @@ class ServiceIpTable extends ServiceBase { // so the reachability-first escalation is never silently coalesced away. private pendingDomainFailureRerun = new Set(); + // A signed config can change while a probing round is in flight. Keep the + // original trigger so the completed stale round can be discarded and one + // fresh round can run against the newly endorsed endpoint set. + private pendingConfigChangeRerun = new Map< + string, + 'domain_failure' | 'ip_failure' | 'periodic' + >(); + @backgroundMethod() async selectBestEndpointForDomain( domain: string, @@ -743,10 +754,16 @@ class ServiceIpTable extends ServiceBase { opts, ).finally(() => { this.speedTestInFlight.delete(domain); + const configChangeTrigger = this.pendingConfigChangeRerun.get(domain); + this.pendingConfigChangeRerun.delete(domain); if (this.pendingDomainFailureRerun.delete(domain)) { void this.selectBestEndpointForDomain(domain, { trigger: 'domain_failure', }); + } else if (configChangeTrigger) { + void this.selectBestEndpointForDomain(domain, { + trigger: configChangeTrigger, + }); } }); this.speedTestInFlight.set(domain, { promise, trigger }); @@ -779,6 +796,9 @@ class ServiceIpTable extends ServiceBase { return; } + const startConfigVersion = configWithRuntime.config.version; + const startConfigHash = computeIpTableConfigHash(configWithRuntime.config); + // A probing round can take tens of seconds; capture the selection // generation so a stale result cannot overwrite a fast failover that // happened while we were measuring. @@ -853,6 +873,38 @@ class ServiceIpTable extends ServiceBase { const bestMeasuredIp = Object.entries(ipLatencies) .filter(([, latency]) => latency !== Infinity) .toSorted((a, b) => a[1] - b[1])[0]?.[0]; + + // The signed config may have been replaced while the probes were + // running. Re-read it immediately before persisting any result: an IP + // removed by the new envelope must never be resurrected as lastBestIp + // or selection by an old round. Queue exactly one fresh round against + // the current config instead. + const latestConfigWithRuntime = await this.getConfig(); + const latestDomainConfig = latestConfigWithRuntime.config.domains[domain]; + const latestConfigHash = computeIpTableConfigHash( + latestConfigWithRuntime.config, + ); + const latestEndpointIps = new Set( + latestDomainConfig?.endpoints.map((endpoint) => endpoint.ip) ?? [], + ); + const measuredEndpointRemoved = [...ipResults.keys()].some( + (ip) => !latestEndpointIps.has(ip), + ); + const selectedCandidateRemoved = + decision.action === 'select_ip' && !latestEndpointIps.has(decision.ip); + if ( + latestConfigWithRuntime.config.version !== startConfigVersion || + latestConfigHash !== startConfigHash || + measuredEndpointRemoved || + selectedCandidateRemoved + ) { + this.pendingConfigChangeRerun.set(domain, opts?.trigger ?? 'periodic'); + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Discarding stale speed test result for ${domain}: signed config changed while probing (v${startConfigVersion}/${startConfigHash} -> v${latestConfigWithRuntime.config.version}/${latestConfigHash}); queueing fresh round`, + }); + return; + } + if (bestMeasuredIp) { await this.backgroundApi.simpleDb.ipTable.updateLastBestIp( domain, @@ -962,7 +1014,7 @@ class ServiceIpTable extends ServiceBase { domain: string, requestType: 'ip' | 'domain', target: string, - startedAtMs?: number, + requestSequence?: number, ): Promise { // Check if IP Table is enabled if (!(await this.isIpTableEnabled())) { @@ -970,7 +1022,7 @@ class ServiceIpTable extends ServiceBase { } const now = Date.now(); - const outcomeAt = startedAtMs ?? now; + const outcomeSequence = requestSequence ?? nextIpTableRequestSequence(); // Get or initialize domain health stats const stats = this.getDomainHealth(domain); @@ -979,12 +1031,14 @@ class ServiceIpTable extends ServiceBase { const endpointHealth = this.getEndpointHealth(stats, requestType, target); // Outcomes arrive through async callbacks with no ordering guarantee: - // the shared ledger orders them by transport start time, so a failure - // whose request predates the newest applied outcome (e.g. a success - // that landed first) never re-increments the counter. + // the shared ledger orders them by a monotonic sequence allocated at + // transport hand-off, so a failure whose request predates the newest + // applied outcome never re-increments the counter, even within one ms. if ( - applyOutcome(endpointHealth, { ok: false, startedAtMs: outcomeAt }) === - 'stale' + applyOutcome(endpointHealth, { + ok: false, + requestSequence: outcomeSequence, + }) === 'stale' ) { defaultLogger.ipTable.request.info({ info: `[IpTable] Ignoring stale ${requestType} failure for ${domain} (${target}): request predates the newest applied outcome`, @@ -1124,12 +1178,12 @@ class ServiceIpTable extends ServiceBase { // Register request failure callback (handles both IP and domain failures) setReportRequestFailureCallback( - ({ domain, requestType, target, startedAtMs }) => { + ({ domain, requestType, target, requestSequence }) => { void this.reportRequestFailure( domain, requestType, target, - startedAtMs, + requestSequence, ); }, ); @@ -1141,14 +1195,14 @@ class ServiceIpTable extends ServiceBase { // the old failure would create the entry afterwards and count against a // link that already proved healthy. setReportRequestSuccessCallback( - ({ domain, requestType, target, startedAtMs }) => { + ({ domain, requestType, target, requestSequence }) => { const stats = this.getDomainHealth(domain); const endpointHealth = this.getEndpointHealth( stats, requestType, target, ); - applyOutcome(endpointHealth, { ok: true, startedAtMs }); + applyOutcome(endpointHealth, { ok: true, requestSequence }); }, ); diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index 7c55a6e1d762..21ecc66b11fb 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -622,6 +622,9 @@ describe('ipTableAdapter fail-open on domain network failures', () => { }); test('a late failure from a request started before a newer success does not count', async () => { + // Both transports start in the same wall-clock millisecond. Ordering must + // come from the runtime request sequence, not Date.now(). + jest.spyOn(Date, 'now').mockReturnValue(1000); // Request A starts first and fails slowly; request B starts later and // succeeds before A's failure lands. B's success must leave its ledger // mark even though no failure entry existed yet, making A's late @@ -999,7 +1002,7 @@ describe('ipTableAdapter idempotency-gated fallback after SNI started', () => { expect.objectContaining({ requestType: 'ip', target: '93.184.216.34', - startedAtMs: expect.any(Number), + requestSequence: expect.any(Number), }), ); } finally { diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 1a01e8379c70..5d56a7bb7027 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -12,7 +12,11 @@ import { import { getRequestHeaders } from '../Interceptor'; import requestHelper from '../requestHelper'; -import { applyOutcome, createOutcomeLedgerEntry } from './ipTableOutcomeLedger'; +import { + applyOutcome, + createOutcomeLedgerEntry, + nextIpTableRequestSequence, +} from './ipTableOutcomeLedger'; import { isSniFailClosedError } from './sniFailClosedError'; import { redactIpLiterals, safeSniLogValue } from './sniLogRedaction'; import { isProxyActiveForUrl, isSniSupported, sniRequest } from './sniRequest'; @@ -107,12 +111,8 @@ interface IRequestFailureParams { target: string; /** Error message */ error: string; - /** - * Date.now() captured when the request was handed to the transport. - * Outcomes race through async callbacks; consumers order them by request - * start time instead of processing time. - */ - startedAtMs: number; + /** Runtime-local sequence allocated when handed to the transport */ + requestSequence: number; } let reportRequestFailureCallback: @@ -128,8 +128,8 @@ interface IRequestSuccessParams { domain: string; requestType: 'ip' | 'domain'; target: string; - /** Date.now() captured when the request was handed to the transport */ - startedAtMs: number; + /** Runtime-local sequence allocated when handed to the transport */ + requestSequence: number; } let reportRequestSuccessCallback: @@ -243,13 +243,14 @@ const getSelectedIpForHost = memoizee(getSelectedIpForHostInternal, { interface IAdapterFailoverState { /** * Per-hostname outcome ledger (consecutive transport failures + newest - * applied outcome ordered by transport start time). Entries persist across + * applied outcome ordered by runtime request sequence). Entries persist across * successes so a late-arriving old failure can always be recognized as * stale — deleting them would lose the ordering mark. */ hostFailures: Map; failOpenUntil: number; - activatedAt: number; + /** Sequence of the failure request that opened the circuit */ + activatedRequestSequence: number; /** Hostnames whose failures opened the circuit; only they may close it early */ activatedHostnames: Set; /** Rotates across activations so a dead builtin endpoint is not retried forever */ @@ -324,7 +325,7 @@ function getFailoverState(lookupDomain: string): IAdapterFailoverState { state = { hostFailures: new Map(), failOpenUntil: 0, - activatedAt: 0, + activatedRequestSequence: 0, activatedHostnames: new Set(), fallbackIpIndex: 0, }; @@ -353,7 +354,7 @@ function deactivateFailover(lookupDomain: string, cause: string): void { action: 'deactivated', }); } - // Reset counters but PRESERVE each hostname's lastOutcomeAt watermark: + // Reset counters but PRESERVE each hostname's lastOutcomeSequence watermark: // clearing entries would let failures from requests started before the // recovery re-apply on fresh entries and reopen a circuit that the link // already proved healthy. @@ -361,7 +362,7 @@ function deactivateFailover(lookupDomain: string, cause: string): void { entry.consecutiveFailures = 0; }); state.failOpenUntil = 0; - state.activatedAt = 0; + state.activatedRequestSequence = 0; state.activatedHostnames.clear(); getSelectedIpForHost.clear(); } @@ -377,22 +378,22 @@ async function recordDomainRequestOutcome(options: { rootDomain: string; hostname: string; ok: boolean; - /** Date.now() captured when the request was handed to the transport */ - startedAtMs: number; + /** Runtime-local sequence allocated when handed to the transport */ + requestSequence: number; error?: unknown; }): Promise { - const { rootDomain, hostname, ok, startedAtMs, error } = options; + const { rootDomain, hostname, ok, requestSequence, error } = options; const lookupDomain = await resolveFailoverLookupDomain(rootDomain); const state = getFailoverState(lookupDomain); - // Both success and failure go through the same start-time-ordered ledger: - // a success records its order even on a fresh entry (so an older failure - // arriving later is recognized as stale), and a failure whose request - // started before the newest applied outcome never counts. + // Both success and failure go through the same request-sequence-ordered + // ledger: a success records its order even on a fresh entry (so an older + // failure arriving later is recognized as stale), and a failure whose + // request predates the newest applied outcome never counts. const ledger = getHostOutcomeLedger(state, hostname); if (ok) { - const applied = applyOutcome(ledger, { ok: true, startedAtMs }); + const applied = applyOutcome(ledger, { ok: true, requestSequence }); // A success only proves the path that was exercised: it may close the // circuit only when it comes from a hostname that opened it AND started // after activation (a long-in-flight request resolving late says @@ -401,7 +402,7 @@ async function recordDomainRequestOutcome(options: { applied === 'applied' && state.failOpenUntil > Date.now() && state.activatedHostnames.has(hostname) && - startedAtMs >= state.activatedAt + requestSequence > state.activatedRequestSequence ) { deactivateFailover(lookupDomain, 'domain_recovered'); } @@ -415,7 +416,7 @@ async function recordDomainRequestOutcome(options: { return; } - if (applyOutcome(ledger, { ok: false, startedAtMs }) === 'stale') { + if (applyOutcome(ledger, { ok: false, requestSequence }) === 'stale') { return; } @@ -424,7 +425,7 @@ async function recordDomainRequestOutcome(options: { state.activatedHostnames.add(hostname); if (state.failOpenUntil <= now) { state.failOpenUntil = now + IP_TABLE_ADAPTER_FAILOVER_TTL_MS; - state.activatedAt = now; + state.activatedRequestSequence = requestSequence; state.fallbackIpIndex += 1; getSelectedIpForHost.clear(); logIpTableEvent('warn', 'adapter_failover_activated', { @@ -836,7 +837,7 @@ export function createIpTableAdapter( rootDomain?: string; }): Promise => { const { config, isFallback = false, hostname, rootDomain } = options; - const startedAtMs = Date.now(); + const requestSequence = nextIpTableRequestSequence(); debugLog('[IpTableAdapter] About to call original adapter...'); debugLog( '[IpTableAdapter] Original adapter type:', @@ -901,14 +902,14 @@ export function createIpTableAdapter( rootDomain, hostname, ok: true, - startedAtMs, + requestSequence, }); if (reportRequestSuccessCallback) { reportRequestSuccessCallback({ domain: rootDomain, requestType: 'domain', target: hostname, - startedAtMs, + requestSequence, }); } } @@ -933,14 +934,14 @@ export function createIpTableAdapter( rootDomain, hostname, ok: true, - startedAtMs, + requestSequence, }); if (reportRequestSuccessCallback) { reportRequestSuccessCallback({ domain: rootDomain, requestType: 'domain', target: hostname, - startedAtMs, + requestSequence, }); } } else { @@ -951,7 +952,7 @@ export function createIpTableAdapter( rootDomain, hostname, ok: false, - startedAtMs, + requestSequence, error, }); } @@ -976,7 +977,7 @@ export function createIpTableAdapter( requestType: 'domain', target: hostname, error: error instanceof Error ? error.message : String(error), - startedAtMs, + requestSequence, }); } @@ -1222,7 +1223,7 @@ export function createIpTableAdapter( requestBody ? requestBody.substring(0, 200) : 'null', ); - const sniStartedAtMs = Date.now(); + const sniRequestSequence = nextIpTableRequestSequence(); // One SNI attempt must produce at most one ip-failure report: the // null-response branch throws for non-idempotent requests and that // throw lands in the same catch below, which would otherwise report the @@ -1238,7 +1239,7 @@ export function createIpTableAdapter( requestType: 'ip', target: selectedIp, error: errorMessage, - startedAtMs: sniStartedAtMs, + requestSequence: sniRequestSequence, }); }; @@ -1291,7 +1292,7 @@ export function createIpTableAdapter( domain: rootDomain, requestType: 'ip', target: selectedIp, - startedAtMs: sniStartedAtMs, + requestSequence: sniRequestSequence, }); } diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts index aa4cb24004c9..419cc0823637 100644 --- a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts @@ -1,4 +1,8 @@ -import { applyOutcome, createOutcomeLedgerEntry } from './ipTableOutcomeLedger'; +import { + applyOutcome, + createOutcomeLedgerEntry, + nextIpTableRequestSequence, +} from './ipTableOutcomeLedger'; describe('ipTableOutcomeLedger', () => { it('regression: a newer success applied first blocks an older failure arriving late', () => { @@ -6,46 +10,62 @@ describe('ipTableOutcomeLedger', () => { // succeeds first. B must leave a ledger mark even though the entry was // fresh, so A's late failure is rejected as stale. const entry = createOutcomeLedgerEntry(); - expect(applyOutcome(entry, { ok: true, startedAtMs: 5 })).toBe('applied'); - expect(applyOutcome(entry, { ok: false, startedAtMs: 1 })).toBe('stale'); + expect(applyOutcome(entry, { ok: true, requestSequence: 5 })).toBe( + 'applied', + ); + expect(applyOutcome(entry, { ok: false, requestSequence: 1 })).toBe( + 'stale', + ); expect(entry.consecutiveFailures).toBe(0); - expect(entry.lastOutcomeAt).toBe(5); + expect(entry.lastOutcomeSequence).toBe(5); }); it('old failure processed first is overridden by a newer success', () => { const entry = createOutcomeLedgerEntry(); - expect(applyOutcome(entry, { ok: false, startedAtMs: 1 })).toBe('applied'); + expect(applyOutcome(entry, { ok: false, requestSequence: 1 })).toBe( + 'applied', + ); expect(entry.consecutiveFailures).toBe(1); - expect(applyOutcome(entry, { ok: true, startedAtMs: 5 })).toBe('applied'); + expect(applyOutcome(entry, { ok: true, requestSequence: 5 })).toBe( + 'applied', + ); expect(entry.consecutiveFailures).toBe(0); }); it('both interleavings converge to the same final state', () => { const a = createOutcomeLedgerEntry(); - applyOutcome(a, { ok: true, startedAtMs: 5 }); - applyOutcome(a, { ok: false, startedAtMs: 1 }); + applyOutcome(a, { ok: true, requestSequence: 5 }); + applyOutcome(a, { ok: false, requestSequence: 1 }); const b = createOutcomeLedgerEntry(); - applyOutcome(b, { ok: false, startedAtMs: 1 }); - applyOutcome(b, { ok: true, startedAtMs: 5 }); + applyOutcome(b, { ok: false, requestSequence: 1 }); + applyOutcome(b, { ok: true, requestSequence: 5 }); expect(a).toEqual(b); }); it('consecutive failures accumulate only for non-stale outcomes', () => { const entry = createOutcomeLedgerEntry(); - applyOutcome(entry, { ok: false, startedAtMs: 10 }); - applyOutcome(entry, { ok: false, startedAtMs: 11 }); + applyOutcome(entry, { ok: false, requestSequence: 10 }); + applyOutcome(entry, { ok: false, requestSequence: 11 }); // stale failure from before the first one - applyOutcome(entry, { ok: false, startedAtMs: 5 }); + applyOutcome(entry, { ok: false, requestSequence: 5 }); expect(entry.consecutiveFailures).toBe(2); - expect(entry.lastOutcomeAt).toBe(11); + expect(entry.lastOutcomeSequence).toBe(11); }); - it('equal start times apply in processing order (documented tie semantics)', () => { + it('orders requests that start in the same wall-clock millisecond', () => { + jest.spyOn(Date, 'now').mockReturnValue(5); + const earlierSequence = nextIpTableRequestSequence(); + const laterSequence = nextIpTableRequestSequence(); + expect(Date.now()).toBe(5); + expect(laterSequence).toBeGreaterThan(earlierSequence); + const entry = createOutcomeLedgerEntry(); - applyOutcome(entry, { ok: true, startedAtMs: 5 }); - expect(applyOutcome(entry, { ok: false, startedAtMs: 5 })).toBe('applied'); - expect(entry.consecutiveFailures).toBe(1); + applyOutcome(entry, { ok: true, requestSequence: laterSequence }); + expect( + applyOutcome(entry, { ok: false, requestSequence: earlierSequence }), + ).toBe('stale'); + expect(entry.consecutiveFailures).toBe(0); }); }); diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts index 845263054fc7..42605c81b6c4 100644 --- a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts @@ -4,31 +4,39 @@ * * Request outcomes arrive through async callbacks with no ordering guarantee: * an early-started request can fail slowly while a later-started request - * succeeds first. Ordering by transport start time (`startedAtMs`) — never by - * completion/processing time — makes every interleaving converge to the same - * state: an outcome whose request started before the newest applied outcome - * is stale and rejected, and both success and failure record their order even - * when the entry was previously empty. + * succeeds first. `Date.now()` is not a sufficient ordering key because two + * transports can start in the same millisecond. Every runtime therefore + * assigns a strictly increasing request sequence at transport hand-off; + * ordering by that sequence — never by completion/processing time — makes + * every interleaving converge to the same state. */ export interface IIpTableOutcomeLedgerEntry { /** Consecutive failure count; reset to 0 by an applied success */ consecutiveFailures: number; - /** Transport start time (Date.now) of the newest applied outcome */ - lastOutcomeAt: number; + /** Runtime-local request sequence of the newest applied outcome */ + lastOutcomeSequence: number; } export function createOutcomeLedgerEntry(): IIpTableOutcomeLedgerEntry { - return { consecutiveFailures: 0, lastOutcomeAt: 0 }; + return { consecutiveFailures: 0, lastOutcomeSequence: 0 }; +} + +let ipTableRequestSequence = 0; + +/** Allocate at transport hand-off, before awaiting any network work. */ +export function nextIpTableRequestSequence(): number { + ipTableRequestSequence += 1; + return ipTableRequestSequence; } export function applyOutcome( entry: IIpTableOutcomeLedgerEntry, - outcome: { ok: boolean; startedAtMs: number }, + outcome: { ok: boolean; requestSequence: number }, ): 'applied' | 'stale' { - if (outcome.startedAtMs < entry.lastOutcomeAt) { + if (outcome.requestSequence < entry.lastOutcomeSequence) { return 'stale'; } - entry.lastOutcomeAt = outcome.startedAtMs; + entry.lastOutcomeSequence = outcome.requestSequence; if (outcome.ok) { entry.consecutiveFailures = 0; } else { From 9a253c86e19a3a8468adf7856f172a53c5569136 Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 21 Jul 2026 23:30:30 +0800 Subject: [PATCH 21/22] fix: close ip table review races --- .../entity/SimpleDbEntityIpTable.test.ts | 107 ++++++++++++++++++ .../simple/entity/SimpleDbEntityIpTable.ts | 86 +++++++++++++- .../services/ServiceIpTable.review.test.ts | 23 ++-- .../kit-bg/src/services/ServiceIpTable.ts | 81 ++++++------- .../request/helpers/ipTableAdapter.test.ts | 73 ++++++++++++ .../src/request/helpers/ipTableAdapter.ts | 33 ++++-- .../helpers/ipTableOutcomeLedger.test.ts | 26 ++++- .../request/helpers/ipTableOutcomeLedger.ts | 46 +++++--- 8 files changed, 393 insertions(+), 82 deletions(-) create mode 100644 packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.test.ts diff --git a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.test.ts b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.test.ts new file mode 100644 index 000000000000..c8a34a676f81 --- /dev/null +++ b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.test.ts @@ -0,0 +1,107 @@ +import type { IIpTableRemoteConfig } from '@onekeyhq/shared/src/request/types/ipTable'; +import { computeIpTableConfigHash } from '@onekeyhq/shared/src/utils/ipTableUtils'; + +import { + type ISimpleDbIpTableData, + SimpleDbEntityIpTable, +} from './SimpleDbEntityIpTable'; + +const DOMAIN = 'onekeycn.com'; + +function buildConfig(ip: string): IIpTableRemoteConfig { + return { + version: 1, + ttl_sec: 300, + generated_at: '2026-07-21T00:00:00.000Z', + signature: `signature-${ip}`, + domains: { + [DOMAIN]: { + endpoints: [ + { + ip, + provider: 'test', + region: 'ALL', + weight: 1, + }, + ], + }, + }, + }; +} + +function setupEntity(initial: ISimpleDbIpTableData) { + const entity = new SimpleDbEntityIpTable(); + let store = initial; + jest.spyOn(entity, 'setRawData').mockImplementation(async (builder) => { + store = + typeof builder === 'function' + ? await builder(store) + : (builder as ISimpleDbIpTableData); + return store; + }); + return { entity, getStore: () => store }; +} + +describe('SimpleDbEntityIpTable speed-test commit', () => { + it('rejects a stale result inside the same setRawData critical section', async () => { + const originalConfig = buildConfig('1.1.1.1'); + const replacementConfig = buildConfig('2.2.2.2'); + const initial: ISimpleDbIpTableData = { + config: replacementConfig, + runtime: { + enabled: true, + lastUpdated: 1, + lastRegionCheck: 0, + selections: {}, + }, + }; + const { entity, getStore } = setupEntity(initial); + + await expect( + entity.commitSpeedTestResult({ + domain: DOMAIN, + expectedConfigHash: computeIpTableConfigHash(originalConfig), + measuredEndpointIps: ['1.1.1.1'], + lastBestIp: '1.1.1.1', + selection: '1.1.1.1', + }), + ).resolves.toBe('stale_config'); + + expect(getStore()).toBe(initial); + }); + + it('atomically writes last-best and selection for the current config', async () => { + const config = buildConfig('1.1.1.1'); + const { entity, getStore } = setupEntity({ config }); + + await expect( + entity.commitSpeedTestResult({ + domain: DOMAIN, + expectedConfigHash: computeIpTableConfigHash(config), + measuredEndpointIps: ['1.1.1.1'], + lastBestIp: '1.1.1.1', + selection: '1.1.1.1', + }), + ).resolves.toBe('applied'); + + expect(getStore().runtime?.lastBestIp?.[DOMAIN]).toBe('1.1.1.1'); + expect(getStore().runtime?.selections[DOMAIN]).toBe('1.1.1.1'); + }); + + it('rejects a candidate outside the current endpoint set even when the hash matches', async () => { + const config = buildConfig('2.2.2.2'); + const initial: ISimpleDbIpTableData = { config }; + const { entity, getStore } = setupEntity(initial); + + await expect( + entity.commitSpeedTestResult({ + domain: DOMAIN, + expectedConfigHash: computeIpTableConfigHash(config), + measuredEndpointIps: ['1.1.1.1'], + lastBestIp: '1.1.1.1', + }), + ).resolves.toBe('stale_config'); + + expect(getStore()).toBe(initial); + }); +}); diff --git a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts index 2821e1ca6def..414ad544ab4b 100644 --- a/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts +++ b/packages/kit-bg/src/dbs/simple/entity/SimpleDbEntityIpTable.ts @@ -5,10 +5,15 @@ import type { IIpTableRemoteConfig, IIpTableRuntime, } from '@onekeyhq/shared/src/request/types/ipTable'; -import { pruneIpTableRuntimeSelections } from '@onekeyhq/shared/src/utils/ipTableUtils'; +import { + computeIpTableConfigHash, + pruneIpTableRuntimeSelections, +} from '@onekeyhq/shared/src/utils/ipTableUtils'; import { SimpleDbEntityBase } from '../base/SimpleDbEntityBase'; +const STALE_SPEED_TEST_CONFIG = new Error('stale_speed_test_config'); + /** * IP Table SimpleDB storage structure * Stores CDN config and runtime state @@ -164,6 +169,85 @@ export class SimpleDbEntityIpTable extends SimpleDbEntityBase { + try { + await this.setRawData((data) => { + const currentConfig = data?.config ?? DEFAULT_IP_TABLE_CONFIG; + const currentEndpoints = new Set( + currentConfig.domains[options.domain]?.endpoints.map( + (endpoint) => endpoint.ip, + ) ?? [], + ); + const candidateIps = new Set(options.measuredEndpointIps); + if (options.lastBestIp) { + candidateIps.add(options.lastBestIp); + } + if (options.selection) { + candidateIps.add(options.selection); + } + if ( + computeIpTableConfigHash(currentConfig) !== + options.expectedConfigHash || + [...candidateIps].some((ip) => !currentEndpoints.has(ip)) + ) { + throw STALE_SPEED_TEST_CONFIG; + } + + const runtime = data?.runtime ?? { + enabled: true, + lastUpdated: 0, + lastRegionCheck: 0, + selections: {}, + }; + return { + ...data, + config: data?.config ?? null, + currentRegion: data?.currentRegion ?? 'AUTO', + runtime: { + ...runtime, + ...(options.lastBestIp + ? { + lastBestIp: { + ...runtime.lastBestIp, + [options.domain]: options.lastBestIp, + }, + } + : {}), + ...(options.selection !== undefined + ? { + selections: { + ...runtime.selections, + [options.domain]: options.selection, + }, + } + : {}), + }, + version: data?.version ?? 1, + }; + }); + return 'applied'; + } catch (error) { + if (error === STALE_SPEED_TEST_CONFIG) { + return 'stale_config'; + } + throw error; + } + } + @backgroundMethod() async setEnabled(enabled: boolean): Promise { await this.setRawData((data) => ({ diff --git a/packages/kit-bg/src/services/ServiceIpTable.review.test.ts b/packages/kit-bg/src/services/ServiceIpTable.review.test.ts index 79f717040df4..7209eb907eb0 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.review.test.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.review.test.ts @@ -91,6 +91,7 @@ function buildConfig(ip: string, version = 1): IIpTableRemoteConfig { function createService() { const ipTableDb = { getConfig: jest.fn(), + commitSpeedTestResult: jest.fn(async () => 'applied'), updateLastBestIp: jest.fn(async () => undefined), updateSelection: jest.fn(async () => undefined), }; @@ -125,22 +126,20 @@ describe('ServiceIpTable review regressions', () => { }, ); - it('discards and queues a rerun when the signed config changes during a speed test', async () => { + it('discards and queues a rerun when the atomic commit sees a new signed config', async () => { const { service, ipTableDb } = createService(); const originalConfig = buildConfig('1.1.1.1', 1); - // Keep the numeric version unchanged: the signed payload hash and the - // currently endorsed endpoint set must still invalidate the old round. - const replacementConfig = buildConfig('2.2.2.2', 1); jest.spyOn(service as any, 'isIpTableEnabled').mockResolvedValue(true); jest .spyOn(service as any, 'testMultipleTimes') .mockResolvedValueOnce(100) .mockResolvedValueOnce(20); - jest - .spyOn(service, 'getConfig') - .mockResolvedValueOnce({ config: originalConfig, runtime: undefined }) - .mockResolvedValueOnce({ config: replacementConfig, runtime: undefined }); + jest.spyOn(service, 'getConfig').mockResolvedValueOnce({ + config: originalConfig, + runtime: undefined, + }); + ipTableDb.commitSpeedTestResult.mockResolvedValueOnce('stale_config'); await (service as any).selectBestEndpointForDomainInternal(DOMAIN, { trigger: 'periodic', @@ -148,6 +147,14 @@ describe('ServiceIpTable review regressions', () => { expect(ipTableDb.updateLastBestIp).not.toHaveBeenCalled(); expect(ipTableDb.updateSelection).not.toHaveBeenCalled(); + expect(ipTableDb.commitSpeedTestResult).toHaveBeenCalledWith( + expect.objectContaining({ + domain: DOMAIN, + measuredEndpointIps: ['1.1.1.1'], + lastBestIp: '1.1.1.1', + selection: '1.1.1.1', + }), + ); expect((service as any).pendingConfigChangeRerun.get(DOMAIN)).toBe( 'periodic', ); diff --git a/packages/kit-bg/src/services/ServiceIpTable.ts b/packages/kit-bg/src/services/ServiceIpTable.ts index 96406d2afecd..aa4bb9ac12f3 100644 --- a/packages/kit-bg/src/services/ServiceIpTable.ts +++ b/packages/kit-bg/src/services/ServiceIpTable.ts @@ -68,12 +68,10 @@ interface IEndpointHealth { consecutiveFailures: number; /** Timestamp of last failure */ lastFailureTime: number; - /** - * Runtime-local request sequence of the newest outcome applied to this - * endpoint. Success/failure reports race through async callbacks; an older - * request sequence is stale and ignored even when Date.now() tied. - */ - lastOutcomeSequence: number; + /** Runtime-local request sequence of the latest successful request */ + latestSuccessSequence: number; + /** Failure request sequences newer than latestSuccessSequence */ + failureSequences: Set; } /** @@ -179,7 +177,8 @@ class ServiceIpTable extends ServiceBase { failureCount: 0, consecutiveFailures: 0, lastFailureTime: 0, - lastOutcomeSequence: 0, + latestSuccessSequence: 0, + failureSequences: new Set(), }; bucket.set(target, endpointHealth); } @@ -874,44 +873,6 @@ class ServiceIpTable extends ServiceBase { .filter(([, latency]) => latency !== Infinity) .toSorted((a, b) => a[1] - b[1])[0]?.[0]; - // The signed config may have been replaced while the probes were - // running. Re-read it immediately before persisting any result: an IP - // removed by the new envelope must never be resurrected as lastBestIp - // or selection by an old round. Queue exactly one fresh round against - // the current config instead. - const latestConfigWithRuntime = await this.getConfig(); - const latestDomainConfig = latestConfigWithRuntime.config.domains[domain]; - const latestConfigHash = computeIpTableConfigHash( - latestConfigWithRuntime.config, - ); - const latestEndpointIps = new Set( - latestDomainConfig?.endpoints.map((endpoint) => endpoint.ip) ?? [], - ); - const measuredEndpointRemoved = [...ipResults.keys()].some( - (ip) => !latestEndpointIps.has(ip), - ); - const selectedCandidateRemoved = - decision.action === 'select_ip' && !latestEndpointIps.has(decision.ip); - if ( - latestConfigWithRuntime.config.version !== startConfigVersion || - latestConfigHash !== startConfigHash || - measuredEndpointRemoved || - selectedCandidateRemoved - ) { - this.pendingConfigChangeRerun.set(domain, opts?.trigger ?? 'periodic'); - defaultLogger.ipTable.request.warn({ - info: `[IpTable] Discarding stale speed test result for ${domain}: signed config changed while probing (v${startConfigVersion}/${startConfigHash} -> v${latestConfigWithRuntime.config.version}/${latestConfigHash}); queueing fresh round`, - }); - return; - } - - if (bestMeasuredIp) { - await this.backgroundApi.simpleDb.ipTable.updateLastBestIp( - domain, - bestMeasuredIp, - ); - } - defaultLogger.ipTable.request.info({ info: `[IpTable] Endpoint decision for ${domain}: ${decision.action} (${ decision.reason @@ -935,8 +896,30 @@ class ServiceIpTable extends ServiceBase { return; } + let selection: string | undefined; + if (decision.action === 'select_ip') { + selection = decision.ip; + } else if (decision.action === 'select_domain') { + selection = ''; + } + const commitResult = + await this.backgroundApi.simpleDb.ipTable.commitSpeedTestResult({ + domain, + expectedConfigHash: startConfigHash, + measuredEndpointIps: [...ipResults.keys()], + lastBestIp: bestMeasuredIp, + selection, + }); + if (commitResult === 'stale_config') { + this.pendingConfigChangeRerun.set(domain, opts?.trigger ?? 'periodic'); + defaultLogger.ipTable.request.warn({ + info: `[IpTable] Discarding stale speed test result for ${domain}: signed config changed after probing (expected v${startConfigVersion}/${startConfigHash}); queueing fresh round`, + }); + return; + } + if (decision.action === 'select_ip') { - await this.applySelection(domain, decision.ip); + this.bumpSelectionGeneration(domain); if ((currentSelection ?? '') === '') { defaultLogger.ipTable.metrics.endpointSwitched({ domain, @@ -947,7 +930,7 @@ class ServiceIpTable extends ServiceBase { }); } } else if (decision.action === 'select_domain') { - await this.applySelection(domain, ''); + this.bumpSelectionGeneration(domain); if (currentSelection) { defaultLogger.ipTable.metrics.endpointSwitched({ domain, @@ -1041,7 +1024,7 @@ class ServiceIpTable extends ServiceBase { }) === 'stale' ) { defaultLogger.ipTable.request.info({ - info: `[IpTable] Ignoring stale ${requestType} failure for ${domain} (${target}): request predates the newest applied outcome`, + info: `[IpTable] Ignoring stale ${requestType} failure for ${domain} (${target}): request predates the latest success`, }); return; } @@ -1114,9 +1097,11 @@ class ServiceIpTable extends ServiceBase { // Reset all consecutive failure counters for this domain (they'll be recounted after speed test) stats.directHosts.forEach((health) => { health.consecutiveFailures = 0; + health.failureSequences.clear(); }); stats.ipEndpoints.forEach((health) => { health.consecutiveFailures = 0; + health.failureSequences.clear(); }); // Trigger speed test to find and switch to better endpoint diff --git a/packages/shared/src/request/helpers/ipTableAdapter.test.ts b/packages/shared/src/request/helpers/ipTableAdapter.test.ts index 21ecc66b11fb..2e9e6f7e276a 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.test.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.test.ts @@ -694,6 +694,79 @@ describe('ipTableAdapter fail-open on domain network failures', () => { expect(mockedSniRequest).not.toHaveBeenCalled(); }); + test('a late middle success retracts a fail-open based on non-consecutive failures', async () => { + // Request order: fail 1, success 2, fail 3, fail 4. Completion order: + // fail 1, fail 3, fail 4, success 2. The first three completions can open + // the circuit provisionally, but success 2 later cuts off failure 1 and + // leaves only failures 3 and 4, below the threshold. + let releaseLateSuccess: (() => void) | undefined; + const lateSuccessGate = new Promise((resolve) => { + releaseLateSuccess = resolve; + }); + fallbackAdapter.mockImplementation(async (config) => { + if (config.url?.includes('late-success')) { + await lateSuccessGate; + return { + data: { lateSuccess: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + if (config.url?.includes('direct-ok')) { + return { + data: { direct: true }, + status: 200, + statusText: 'OK', + headers: {}, + config, + request: {}, + }; + } + throw networkError(); + }); + + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/fail-1'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + const lateSuccess = createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/late-success'), + ); + await Promise.resolve(); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/fail-3'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/fail-4'), + ), + ).rejects.toMatchObject({ code: 'ECONNABORTED' }); + + releaseLateSuccess?.(); + await expect(lateSuccess).resolves.toMatchObject({ + data: { lateSuccess: true }, + }); + + mockedSniRequest.mockResolvedValue({ + statusCode: 200, + statusText: 'OK', + headers: {}, + body: '{"sni":true}', + }); + await expect( + createIpTableAdapter({})( + buildConfig('https://wallet.onekeycn.com/wallet/v1/direct-ok'), + ), + ).resolves.toMatchObject({ data: { direct: true } }); + expect(mockedSniRequest).not.toHaveBeenCalled(); + }); + test('after recovery closes the circuit, old failures landing later stay stale', async () => { // Three slow requests start BEFORE anything goes wrong. The circuit then // opens, recovers (which must preserve the per-hostname watermark), and diff --git a/packages/shared/src/request/helpers/ipTableAdapter.ts b/packages/shared/src/request/helpers/ipTableAdapter.ts index 5d56a7bb7027..19b2753fc2f3 100644 --- a/packages/shared/src/request/helpers/ipTableAdapter.ts +++ b/packages/shared/src/request/helpers/ipTableAdapter.ts @@ -354,11 +354,12 @@ function deactivateFailover(lookupDomain: string, cause: string): void { action: 'deactivated', }); } - // Reset counters but PRESERVE each hostname's lastOutcomeSequence watermark: - // clearing entries would let failures from requests started before the - // recovery re-apply on fresh entries and reopen a circuit that the link - // already proved healthy. + // Reset unresolved failures but preserve each hostname's latest success + // sequence: clearing entries would let failures from requests started + // before recovery re-apply on fresh entries and reopen a circuit that the + // link already proved healthy. state.hostFailures.forEach((entry) => { + entry.failureSequences.clear(); entry.consecutiveFailures = 0; }); state.failOpenUntil = 0; @@ -387,13 +388,31 @@ async function recordDomainRequestOutcome(options: { const state = getFailoverState(lookupDomain); // Both success and failure go through the same request-sequence-ordered - // ledger: a success records its order even on a fresh entry (so an older - // failure arriving later is recognized as stale), and a failure whose - // request predates the newest applied outcome never counts. + // ledger. A success removes all failures no later than itself while keeping + // later-request failures, so completion order cannot change the count. const ledger = getHostOutcomeLedger(state, hostname); if (ok) { const applied = applyOutcome(ledger, { ok: true, requestSequence }); + // A success may arrive after later failures and reveal that the failures + // which opened the circuit were not actually consecutive in request + // order. Retract that provisional activation once every activating host + // falls below the threshold; this is ordering reconciliation, not a claim + // that an older request proves present-time recovery. + const activationWasInvalidated = + applied === 'applied' && + state.failOpenUntil > Date.now() && + state.activatedHostnames.has(hostname) && + ledger.consecutiveFailures < IP_TABLE_DOMAIN_FAILOVER_THRESHOLD && + [...state.activatedHostnames].every( + (activatedHostname) => + (state.hostFailures.get(activatedHostname)?.consecutiveFailures ?? + 0) < IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, + ); + if (activationWasInvalidated) { + deactivateFailover(lookupDomain, 'failure_sequence_reconciled'); + return; + } // A success only proves the path that was exercised: it may close the // circuit only when it comes from a hostname that opened it AND started // after activation (a long-in-flight request resolving late says diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts index 419cc0823637..fcea1ebbf609 100644 --- a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts @@ -17,7 +17,8 @@ describe('ipTableOutcomeLedger', () => { 'stale', ); expect(entry.consecutiveFailures).toBe(0); - expect(entry.lastOutcomeSequence).toBe(5); + expect(entry.latestSuccessSequence).toBe(5); + expect(entry.failureSequences).toEqual(new Set()); }); it('old failure processed first is overridden by a newer success', () => { @@ -44,14 +45,29 @@ describe('ipTableOutcomeLedger', () => { expect(a).toEqual(b); }); - it('consecutive failures accumulate only for non-stale outcomes', () => { + it('keeps only failures after a late-arriving success in request order', () => { + // Request order is failure 1, success 2, failure 3, but completion order + // is failure 1, failure 3, success 2. The success must still cut off the + // first failure while preserving the third. + const entry = createOutcomeLedgerEntry(); + applyOutcome(entry, { ok: false, requestSequence: 1 }); + applyOutcome(entry, { ok: false, requestSequence: 3 }); + expect(applyOutcome(entry, { ok: true, requestSequence: 2 })).toBe( + 'applied', + ); + expect(entry.consecutiveFailures).toBe(1); + }); + + it('accumulates unresolved failures regardless of completion order', () => { const entry = createOutcomeLedgerEntry(); applyOutcome(entry, { ok: false, requestSequence: 10 }); applyOutcome(entry, { ok: false, requestSequence: 11 }); - // stale failure from before the first one + // This request completed last but is still an unresolved failure: without + // a later success in request order, it must remain in the set. applyOutcome(entry, { ok: false, requestSequence: 5 }); - expect(entry.consecutiveFailures).toBe(2); - expect(entry.lastOutcomeSequence).toBe(11); + expect(entry.consecutiveFailures).toBe(3); + expect(entry.latestSuccessSequence).toBe(0); + expect(entry.failureSequences).toEqual(new Set([10, 11, 5])); }); it('orders requests that start in the same wall-clock millisecond', () => { diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts index 42605c81b6c4..7f658630f394 100644 --- a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts @@ -7,18 +7,25 @@ * succeeds first. `Date.now()` is not a sufficient ordering key because two * transports can start in the same millisecond. Every runtime therefore * assigns a strictly increasing request sequence at transport hand-off; - * ordering by that sequence — never by completion/processing time — makes - * every interleaving converge to the same state. + * ordering by that sequence — never by completion/processing time — lets a + * success remove every failure at or before its request position while + * retaining failures from later requests, regardless of arrival order. */ export interface IIpTableOutcomeLedgerEntry { - /** Consecutive failure count; reset to 0 by an applied success */ + /** Number of unresolved failures after the latest success */ consecutiveFailures: number; - /** Runtime-local request sequence of the newest applied outcome */ - lastOutcomeSequence: number; + /** Runtime-local request sequence of the latest successful request */ + latestSuccessSequence: number; + /** Failure request sequences newer than latestSuccessSequence */ + failureSequences: Set; } export function createOutcomeLedgerEntry(): IIpTableOutcomeLedgerEntry { - return { consecutiveFailures: 0, lastOutcomeSequence: 0 }; + return { + consecutiveFailures: 0, + latestSuccessSequence: 0, + failureSequences: new Set(), + }; } let ipTableRequestSequence = 0; @@ -33,14 +40,27 @@ export function applyOutcome( entry: IIpTableOutcomeLedgerEntry, outcome: { ok: boolean; requestSequence: number }, ): 'applied' | 'stale' { - if (outcome.requestSequence < entry.lastOutcomeSequence) { - return 'stale'; - } - entry.lastOutcomeSequence = outcome.requestSequence; if (outcome.ok) { - entry.consecutiveFailures = 0; - } else { - entry.consecutiveFailures += 1; + if (outcome.requestSequence <= entry.latestSuccessSequence) { + return 'stale'; + } + entry.latestSuccessSequence = outcome.requestSequence; + entry.failureSequences.forEach((failureSequence) => { + if (failureSequence <= outcome.requestSequence) { + entry.failureSequences.delete(failureSequence); + } + }); + entry.consecutiveFailures = entry.failureSequences.size; + return 'applied'; + } + + if ( + outcome.requestSequence <= entry.latestSuccessSequence || + entry.failureSequences.has(outcome.requestSequence) + ) { + return 'stale'; } + entry.failureSequences.add(outcome.requestSequence); + entry.consecutiveFailures = entry.failureSequences.size; return 'applied'; } From ab6620dc7b5c8ea40624e984f2427c50c1c79e9d Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 21 Jul 2026 23:50:43 +0800 Subject: [PATCH 22/22] fix: bound ip table failure ledger --- .../helpers/ipTableOutcomeLedger.test.ts | 48 +++++++++++++++++++ .../request/helpers/ipTableOutcomeLedger.ts | 36 +++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts index fcea1ebbf609..4b7ca2849b47 100644 --- a/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.test.ts @@ -1,4 +1,5 @@ import { + IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP, applyOutcome, createOutcomeLedgerEntry, nextIpTableRequestSequence, @@ -84,4 +85,51 @@ describe('ipTableOutcomeLedger', () => { ).toBe('stale'); expect(entry.consecutiveFailures).toBe(0); }); + + it('bounds retained failures while keeping the greatest request sequences', () => { + const entry = createOutcomeLedgerEntry(); + const highestSequence = IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP + 5; + // Insert the greatest sequence first so insertion-order eviction would + // incorrectly discard it. Retention must be based on request order. + const completionOrder = [ + highestSequence, + ...Array.from({ length: highestSequence - 1 }, (_, index) => index + 1), + ]; + completionOrder.forEach((requestSequence) => { + applyOutcome(entry, { ok: false, requestSequence }); + }); + + expect(entry.failureSequences.size).toBe( + IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP, + ); + expect([...entry.failureSequences].toSorted((a, b) => a - b)).toEqual( + Array.from( + { length: IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP }, + (_, index) => index + 6, + ), + ); + expect(entry.consecutiveFailures).toBe(IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP); + }); + + it('keeps an exact below-threshold count after pruning with a late success', () => { + const entry = createOutcomeLedgerEntry(); + const highestSequence = IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP + 5; + for ( + let requestSequence = 1; + requestSequence <= highestSequence; + requestSequence += 1 + ) { + applyOutcome(entry, { ok: false, requestSequence }); + } + + applyOutcome(entry, { + ok: true, + requestSequence: highestSequence - 2, + }); + + expect(entry.failureSequences).toEqual( + new Set([highestSequence - 1, highestSequence]), + ); + expect(entry.consecutiveFailures).toBe(2); + }); }); diff --git a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts index 7f658630f394..dbb4aefadf58 100644 --- a/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts +++ b/packages/shared/src/request/helpers/ipTableOutcomeLedger.ts @@ -1,3 +1,8 @@ +import { + IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, + IP_TABLE_SNI_FAILURE_THRESHOLD, +} from '../constants/ipTableDefaults'; + /** * Outcome ledger shared by the adapter's per-hostname fail-open tracking and * the service's per-endpoint health stats. @@ -11,12 +16,16 @@ * success remove every failure at or before its request position while * retaining failures from later requests, regardless of arrival order. */ +export const IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP = + Math.max(IP_TABLE_DOMAIN_FAILOVER_THRESHOLD, IP_TABLE_SNI_FAILURE_THRESHOLD) * + 4; + export interface IIpTableOutcomeLedgerEntry { - /** Number of unresolved failures after the latest success */ + /** Capped number of unresolved failures after the latest success */ consecutiveFailures: number; /** Runtime-local request sequence of the latest successful request */ latestSuccessSequence: number; - /** Failure request sequences newer than latestSuccessSequence */ + /** Greatest retained failure sequences newer than latestSuccessSequence */ failureSequences: Set; } @@ -36,6 +45,24 @@ export function nextIpTableRequestSequence(): number { return ipTableRequestSequence; } +function capFailureSequences(entry: IIpTableOutcomeLedgerEntry): void { + while (entry.failureSequences.size > IP_TABLE_OUTCOME_LEDGER_FAILURE_CAP) { + let smallestSequence: number | undefined; + entry.failureSequences.forEach((failureSequence) => { + if ( + smallestSequence === undefined || + failureSequence < smallestSequence + ) { + smallestSequence = failureSequence; + } + }); + if (smallestSequence === undefined) { + break; + } + entry.failureSequences.delete(smallestSequence); + } +} + export function applyOutcome( entry: IIpTableOutcomeLedgerEntry, outcome: { ok: boolean; requestSequence: number }, @@ -61,6 +88,11 @@ export function applyOutcome( return 'stale'; } entry.failureSequences.add(outcome.requestSequence); + // Only threshold comparisons consume this count. Keeping the greatest + // request sequences preserves those comparisons under any later success: + // discarded sequences are older than every retained one, so they can never + // be the difference between below-threshold and at/above-threshold. + capFailureSequences(entry); entry.consecutiveFailures = entry.failureSequences.size; return 'applied'; }