diff --git a/src/storage/version/audit.js b/src/storage/version/audit.js index c51332b7..c876218b 100644 --- a/src/storage/version/audit.js +++ b/src/storage/version/audit.js @@ -270,7 +270,10 @@ export async function writeAuditEntry(env, ctx, repo, fileId, entry, attempt = 0 return { status: resp?.$metadata?.httpStatusCode ?? 200 }; } catch (e) { if (e?.$metadata?.httpStatusCode === 412 && attempt < 4) { - const delay = Math.random() * 50 * (attempt + 1); + // Exponential jitter: 0-50, 0-100, 0-200, 0-400 ms worst-case (~750 ms total). + // Grown from linear (~500 ms total) after PreconditionFailed burst of 5011/24h + // on 2026-05-12. Retry count unchanged. See COR-26. + const delay = Math.random() * 50 * 2 ** attempt; // eslint-disable-next-line no-await-in-loop -- sequential retry with jitter await new Promise((r) => { setTimeout(r, delay); diff --git a/test/storage/version/audit.test.js b/test/storage/version/audit.test.js index 3e8f2d7f..b01f42ff 100644 --- a/test/storage/version/audit.test.js +++ b/test/storage/version/audit.test.js @@ -863,6 +863,89 @@ describe('Version Audit', () => { assert.strictEqual(getCallCount, 5, 'must re-read on each attempt'); }); + it('uses exponential jitter backoff (0-50, 0-100, 0-200, 0-400 ms) across four 412 retries', async () => { + // Stubs setTimeout and Math.random to capture per-retry delays. + // Asserts per-attempt exponential upper bounds and that total elapsed sleep + // exceeds the prior linear worst-case (500 ms). See COR-26. + const originalSetTimeout = globalThis.setTimeout; + const originalRandom = Math.random; + const delays = []; + Math.random = () => 0.999999; + globalThis.setTimeout = (fn, ms) => { + delays.push(ms); + fn(); + return 0; + }; + + try { + const makeBody = () => new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('')); + controller.close(); + }, + }); + + const { writeAuditEntry } = await esmock( + '../../../src/storage/version/audit.js', + { + '@aws-sdk/client-s3': { + S3Client: function S3Client() { + this.send = async (cmd) => { + if (cmd instanceof GetObjectCommand) { + return { Body: makeBody(), ETag: '"etag-x"' }; + } + if (cmd instanceof PutObjectCommand) { + const err = new Error('precondition failed'); + err.$metadata = { httpStatusCode: 412 }; + throw err; + } + return { $metadata: { httpStatusCode: 200 } }; + }; + }, + GetObjectCommand, + PutObjectCommand, + }, + '../../../src/storage/utils/config.js': { default: () => ({}) }, + }, + ); + + await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', { + timestamp: '5000', + users: '[{"email":"u@x.com"}]', + path: 'repo/doc.html', + }); + } finally { + globalThis.setTimeout = originalSetTimeout; + Math.random = originalRandom; + } + + assert.strictEqual(delays.length, 4, 'must sleep between each of the 4 retries'); + const upperBounds = [50, 100, 200, 400]; + delays.forEach((ms, i) => { + assert.ok( + ms >= 0 && ms < upperBounds[i], + `delay[${i}]=${ms} must be within [0, ${upperBounds[i]})`, + ); + }); + assert.ok( + delays[2] >= 150, + `delay[2]=${delays[2]} must exceed the prior linear cap of 150 ms`, + ); + assert.ok( + delays[3] >= 200, + `delay[3]=${delays[3]} must exceed the prior linear cap of 200 ms`, + ); + const total = delays.reduce((a, b) => a + b, 0); + assert.ok( + total > 500, + `total elapsed sleep ${total} must exceed prior linear worst-case (500 ms)`, + ); + assert.ok( + total < 750, + `total elapsed sleep ${total} must stay within the new worst-case (~750 ms)`, + ); + }); + it('succeeds on the 5th attempt after four 412s', async () => { let getCallCount = 0; let putCallCount = 0;