From 72a14e0981281aa309a8f8927bbce95c97ea4fab Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 14:18:34 +0200 Subject: [PATCH 01/48] test: pin store urls and the binary post gate covers daCtx.sourcePath, the /source/... url handed to env.daadmin.fetch across GET/HEAD/POST, and a 415 for a multipart File that is not text/html. red on main, where / reads /.html and /page.html reads /page.html.html. Relates to #256. --- test/routes/da-admin.test.js | 179 ++++++++++++++++++++++++++++++++++- test/utils/daCtx.test.js | 102 ++++++++++++++++++++ 2 files changed, 278 insertions(+), 3 deletions(-) diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index 2824b9c9..9bd9adbf 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -16,7 +16,45 @@ import esmock from 'esmock'; import reqs from '../mocks/req.js'; const { getDaCtx } = await import('../../src/utils/daCtx.js'); -const { daSourceHead } = await import('../../src/routes/da-admin.js'); +const { daSourceHead, daSourcePost } = await import('../../src/routes/da-admin.js'); + +const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer t' } }); + +const formReq = (url, data) => { + const body = new FormData(); + body.set('data', data); + return new Request(url, { method: 'POST', body, headers: { Authorization: 'Bearer t' } }); +}; + +// records every URL handed to env.daadmin.fetch, whether it is called with a +// URL (GET non-HTML, HEAD) or with a Request (GET HTML, POST) +const recorder = () => { + const fetched = []; + const env = { + DA_ADMIN: 'https://admin.da.live', + daadmin: { + fetch: async (input) => { + fetched.push(input instanceof Request ? input.url : input.href); + return new Response('stored', { status: 200 }); + }, + }, + }; + return { env, fetched }; +}; + +const mockRoutes = async () => esmock('../../src/routes/da-admin.js', { + '../../src/utils/aemCtx.js': { + getAemCtx: () => ({}), + getAEMHtml: async () => '', + }, + '../../src/render/compose.js': { + composeHtml: async () => ({ tree: true }), + serializeHtml: () => 'composed', + }, + '../../src/ue/ue.js': { + applyUEInstrumentation: async () => {}, + }, +}); describe('daSourceHead', () => { describe('when no authToken is present', () => { @@ -45,8 +83,6 @@ describe('daSourceGet', () => { daadmin: { fetch: async () => new Response('stored', { status: 200 }) }, }; - const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer t' } }); - // record which composition / instrumentation calls happen and with what let calls; @@ -203,3 +239,140 @@ describe('daSourceGet', () => { assert.ok(html.includes('Unable to retrieve AEM branch')); }); }); + +describe('source URLs', () => { + it('GET / reads /index.html', async () => { + const { daSourceGet } = await mockRoutes(); + const { env, fetched } = recorder(); + const req = authedReq('https://main--site--org.ue.da.live/'); + const daCtx = getDaCtx(req); + + await daSourceGet({ req, env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/index.html']); + }); + + it('GET /page.html reads /page.html', async () => { + const { daSourceGet } = await mockRoutes(); + const { env, fetched } = recorder(); + const req = authedReq('https://main--site--org.ue.da.live/page.html'); + const daCtx = getDaCtx(req); + + await daSourceGet({ req, env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); + + it('GET /Media/Logo.PNG reads /media/logo.png', async () => { + const { daSourceGet } = await mockRoutes(); + const { env, fetched } = recorder(); + const req = authedReq('https://main--site--org.ue.da.live/Media/Logo.PNG'); + const daCtx = getDaCtx(req); + + await daSourceGet({ req, env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/media/logo.png']); + }); + + it('HEAD / reads /index.html', async () => { + const { env, fetched } = recorder(); + const daCtx = getDaCtx(authedReq('https://main--site--org.ue.da.live/')); + + await daSourceHead({ env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/index.html']); + }); + + it('HEAD /page.html reads /page.html', async () => { + const { env, fetched } = recorder(); + const daCtx = getDaCtx(authedReq('https://main--site--org.ue.da.live/page.html')); + + await daSourceHead({ env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); + + it('HEAD /Media/Logo.PNG reads /media/logo.png', async () => { + const { env, fetched } = recorder(); + const daCtx = getDaCtx(authedReq('https://main--site--org.ue.da.live/Media/Logo.PNG')); + + await daSourceHead({ env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/media/logo.png']); + }); + + it('POST / writes /index.html', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'index.html', { type: 'text/html' }); + const req = formReq('https://main--site--org.ue.da.live/', html); + const daCtx = getDaCtx(req); + + await daSourcePost({ req, env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/index.html']); + }); + + it('POST /page.html writes /page.html', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'page.html', { type: 'text/html' }); + const req = formReq('https://main--site--org.ue.da.live/page.html', html); + const daCtx = getDaCtx(req); + + await daSourcePost({ req, env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); + + it('POST /Media/Logo.PNG writes /media/logo.png', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'logo.html', { type: 'text/html' }); + const req = formReq('https://main--site--org.ue.da.live/Media/Logo.PNG', html); + const daCtx = getDaCtx(req); + + await daSourcePost({ req, env, daCtx }); + + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/media/logo.png']); + }); +}); + +describe('daSourcePost', () => { + it('refuses a binary File with 415 and does not write', async () => { + const { env, fetched } = recorder(); + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]); + const png = new File([bytes], 'logo.png', { type: 'image/png' }); + const req = formReq('https://main--site--org.ue.da.live/media/logo.png', png); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 415); + assert.deepStrictEqual(fetched, []); + }); + + it('writes an HTML File', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'page.html', { type: 'text/html' }); + const req = formReq('https://main--site--org.ue.da.live/Page', html); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 200); + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); + + it('returns a response when the content type is not a form type', async () => { + const { env, fetched } = recorder(); + const req = new Request('https://main--site--org.ue.da.live/page', { + method: 'POST', + body: '{}', + headers: { Authorization: 'Bearer t', 'Content-Type': 'application/json' }, + }); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.ok(res instanceof Response); + assert.deepStrictEqual(fetched, []); + }); +}); diff --git a/test/utils/daCtx.test.js b/test/utils/daCtx.test.js index 3311c729..dd515213 100644 --- a/test/utils/daCtx.test.js +++ b/test/utils/daCtx.test.js @@ -136,6 +136,108 @@ describe('DA context', () => { }); }); + describe('sourcePath', () => { + const ctxFor = (pathname) => getDaCtx( + new Request(`https://main--site--org.ue.da.live${pathname}`), + ); + + it('maps / to /index.html', () => { + const ctx = ctxFor('/'); + assert.strictEqual(ctx.sourcePath, '/index.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /index to /index.html', () => { + const ctx = ctxFor('/index'); + assert.strictEqual(ctx.sourcePath, '/index.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /folder/ to /folder/index.html', () => { + const ctx = ctxFor('/folder/'); + assert.strictEqual(ctx.sourcePath, '/folder/index.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /folder to /folder.html', () => { + const ctx = ctxFor('/folder'); + assert.strictEqual(ctx.sourcePath, '/folder.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /page to /page.html', () => { + const ctx = ctxFor('/page'); + assert.strictEqual(ctx.sourcePath, '/page.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /page.html to /page.html', () => { + const ctx = ctxFor('/page.html'); + assert.strictEqual(ctx.sourcePath, '/page.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /Page.HTML to /page.html', () => { + const ctx = ctxFor('/Page.HTML'); + assert.strictEqual(ctx.sourcePath, '/page.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /x.html.html to /x.html.html', () => { + const ctx = ctxFor('/x.html.html'); + assert.strictEqual(ctx.sourcePath, '/x.html.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /Media/Logo.PNG to /media/logo.png', () => { + const ctx = ctxFor('/Media/Logo.PNG'); + assert.strictEqual(ctx.sourcePath, '/media/logo.png'); + assert.strictEqual(ctx.ext, 'png'); + }); + + it('maps /sheet.json to /sheet.json', () => { + const ctx = ctxFor('/sheet.json'); + assert.strictEqual(ctx.sourcePath, '/sheet.json'); + assert.strictEqual(ctx.ext, 'json'); + }); + + it('maps /a/b.plain.html to /a/b.plain.html', () => { + const ctx = ctxFor('/a/b.plain.html'); + assert.strictEqual(ctx.sourcePath, '/a/b.plain.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /a//b to /a/b.html', () => { + const ctx = ctxFor('/a//b'); + assert.strictEqual(ctx.sourcePath, '/a/b.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /.hidden to /.hidden', () => { + const ctx = ctxFor('/.hidden'); + assert.strictEqual(ctx.sourcePath, '/.hidden'); + assert.strictEqual(ctx.ext, 'hidden'); + }); + + it('maps /page. to /page.', () => { + const ctx = ctxFor('/page.'); + assert.strictEqual(ctx.sourcePath, '/page.'); + assert.strictEqual(ctx.ext, ''); + }); + + it('maps /explaining to /explaining.html', () => { + const ctx = ctxFor('/explaining'); + assert.strictEqual(ctx.sourcePath, '/explaining.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + + it('maps /v1.2 to /v1.2', () => { + const ctx = ctxFor('/v1.2'); + assert.strictEqual(ctx.sourcePath, '/v1.2'); + assert.strictEqual(ctx.ext, '2'); + }); + }); + describe('Invalid URL context', async () => { beforeEach(async () => { daCtx = getDaCtx(reqs.invalid); From c2613bb8fd655704699cf4c1be65ef59cc5b8bbb Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 14:27:46 +0200 Subject: [PATCH 02/48] fix: derive daCtx.sourcePath and repoint store urls sourcePath comes from the filename rather than ext, so / reads /index.html and /page.html stops reading /page.html.html. ext is unchanged, so the compose vs raw proxy split at da-admin.js:93 holds. drops six reader-less daCtx fields. daSourcePost now 415s a File whose declared type is not text/html, which would otherwise corrupt a binary body once the write lands on the key GET reads, and returns a response instead of undefined so withCorsHeaders cannot throw. Relates to #256. --- src/responses/index.js | 4 ++++ src/routes/da-admin.js | 31 +++++++++++++++++-------------- src/utils/daCtx.js | 34 +++++++++------------------------- test/utils/daCtx.test.js | 20 +++++++++----------- 4 files changed, 39 insertions(+), 50 deletions(-) diff --git a/src/responses/index.js b/src/responses/index.js index 3bc9e4be..1c34e53e 100644 --- a/src/responses/index.js +++ b/src/responses/index.js @@ -43,6 +43,10 @@ export function get401(message = DEFAULT_UNAUTHORIZED_HTML_MESSAGE) { return daResp({ body: message, status: 401, contentType: 'text/html' }); } +export function get415(message = '') { + return daResp({ body: message, status: 415, contentType: 'text/html' }); +} + export function head401() { return new Response(null, { status: 401 }); } diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index 588d9165..d2d9db87 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -22,12 +22,16 @@ import { applyQuickEditToDocument, buildQuickEditCookie, buildQuickEditNotFoundResponse, } from '../utils/quick-edit.js'; import { - daResp, get401, get404, head401, + daResp, get401, get404, get415, head401, } from '../responses/index.js'; import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, UNAUTHORIZED_HTML_MESSAGE } from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; +// file content types accepted on an HTML POST. formData() reports octet-stream or +// text/plain for a part whose type the client left unset, so both stay allowed. +const HTML_POST_TYPES = ['text/html', 'text/plain', 'application/octet-stream']; + async function getFileBody(data) { const text = await data.text(); return { body: text, type: data.type }; @@ -71,7 +75,7 @@ async function getPageTemplate(env, daCtx, aemCtx) { export async function daSourceGet({ req, env, daCtx }) { const { - org, site, path, ext, authToken, + org, site, sourcePath, ext, authToken, } = daCtx; // check if Authorization header is present @@ -91,11 +95,8 @@ export async function daSourceGet({ req, env, daCtx }) { headers.set('Authorization', authToken); if (ext !== 'html') { - /* - for non-HTML files, simply proxy the request without processing - and ensure that extensions are not duplicated - */ - const adminUrl = new URL(`/source/${org}/${site}${path}`, env.DA_ADMIN); + // for non-HTML files, simply proxy the request without processing + const adminUrl = new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); console.log(`-> ${adminUrl.toString()}`); const response = await env.daadmin.fetch(adminUrl, { method: 'GET', headers }); console.log(`<- ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); @@ -116,7 +117,7 @@ export async function daSourceGet({ req, env, daCtx }) { // get the content from DA admin const adminUrl = new URL( - `/source/${org}/${site}${path}.${ext}`, + `/source/${org}/${site}${sourcePath}`, env.DA_ADMIN, ); @@ -163,7 +164,7 @@ export async function daSourceGet({ req, env, daCtx }) { export async function daSourceHead({ env, daCtx }) { const { - org, site, path, ext, authToken, + org, site, sourcePath, authToken, } = daCtx; if (!authToken) { @@ -173,8 +174,7 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const adminPath = ext !== 'html' ? path : `${path}.${ext}`; - const adminUrl = new URL(`/source/${org}/${site}${adminPath}`, env.DA_ADMIN); + const adminUrl = new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); console.log(`-> HEAD ${adminUrl.toString()}`); const response = await env.daadmin.fetch(adminUrl, { method: 'HEAD', headers }); console.log(`<- HEAD ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); @@ -183,12 +183,15 @@ export async function daSourceHead({ env, daCtx }) { export async function daSourcePost({ req, env, daCtx }) { const { - org, site, path, ext, authToken, + org, site, sourcePath, authToken, } = daCtx; const obj = await putHelper(req, env, daCtx); if (obj && obj.data) { const isFile = obj.data instanceof File; + if (isFile && obj.data.type && !HTML_POST_TYPES.includes(obj.data.type)) { + return get415(); + } const { body: bodyHtml } = isFile ? await getFileBody(obj.data) : getTextBody(obj.data); @@ -212,7 +215,7 @@ export async function daSourcePost({ req, env, daCtx }) { body.set('data', data); const headers = { Authorization: authToken }; const adminUrl = new URL( - `/source/${org}/${site}${path}.${ext}`, + `/source/${org}/${site}${sourcePath}`, env.DA_ADMIN, ); // eslint-disable-next-line no-param-reassign @@ -227,5 +230,5 @@ export async function daSourcePost({ req, env, daCtx }) { return response; } - return undefined; + return get415(); } diff --git a/src/utils/daCtx.js b/src/utils/daCtx.js index 3e509d39..b2cf1338 100644 --- a/src/utils/daCtx.js +++ b/src/utils/daCtx.js @@ -73,8 +73,10 @@ function getSiteToken(req) { /** * Gets Dark Alley Context - * @param {pathname} pathname - * @returns {DaCtx} The Dark Alley Context. + * @param {Request} req The incoming request. + * @returns {Object} The Dark Alley Context, where `path` is the request pathname + * (minus the `/org/site` prefix on localhost), `aemPathname` is the path requested + * from `*.aem.page`, and `sourcePath` is the path requested from the store. */ export function getDaCtx(req) { const { pathname, hostname, searchParams } = new URL(req.url); @@ -98,33 +100,15 @@ export function getDaCtx(req) { // Sanitize the remaining path parts const pathParts = parts.filter((part) => part !== ''); - const keyBase = `${site}/${pathParts.join('/')}`; - // Get the final source name - daCtx.filename = pathParts.pop() || ''; - - // Handle folders and files under a site - const split = daCtx.filename.split('.'); - - // DA Content - Add HTML if there is only one part to the split - if (split.length === 1) split.push('html'); - daCtx.isFile = split.length > 1; - if (daCtx.isFile) daCtx.ext = split.pop(); - daCtx.name = split.join('.'); - - // Set keys - daCtx.key = daCtx?.ext === 'html' ? `${keyBase}.html` : keyBase; - daCtx.propsKey = `${daCtx.key}.props`; + // Get the final source name and its extension + const filename = pathParts.pop() || ''; + const dotted = filename.includes('.'); + daCtx.ext = dotted ? filename.split('.').pop() : 'html'; // Set paths for API consumption daCtx.aemPathname = path.endsWith('/index') ? path.substring(0, path.length - 5) : path; - const daPathBase = [...pathParts, daCtx.name].join('/'); - - if (!daCtx.ext || (!daCtx.name.includes('plain') && daCtx.ext === 'html')) { - daCtx.pathname = `/${daPathBase}`; - } else { - daCtx.pathname = `/${daPathBase}.${daCtx.ext}`; - } + daCtx.sourcePath = `/${[...pathParts, dotted ? filename : `${filename}.html`].join('/')}`; const query = Object.fromEntries(searchParams.entries()); if (typeof query['ue-service'] === 'string') { diff --git a/test/utils/daCtx.test.js b/test/utils/daCtx.test.js index dd515213..96a4a9ca 100644 --- a/test/utils/daCtx.test.js +++ b/test/utils/daCtx.test.js @@ -41,7 +41,7 @@ describe('DA context', () => { }); it('should return the correct path names', () => { - assert.strictEqual(daCtx.pathname, '/folder/content'); + assert.strictEqual(daCtx.sourcePath, '/folder/content.html'); assert.strictEqual(daCtx.aemPathname, '/folder/content'); }); @@ -68,7 +68,7 @@ describe('DA context', () => { }); it('should return the correct path names', () => { - assert.strictEqual(daCtx.pathname, '/folder/content'); + assert.strictEqual(daCtx.sourcePath, '/folder/content.html'); assert.strictEqual(daCtx.aemPathname, '/folder/content'); }); @@ -83,7 +83,7 @@ describe('DA context', () => { }); it('should return the correct path names', () => { - assert.strictEqual(daCtx.pathname, '/index'); + assert.strictEqual(daCtx.sourcePath, '/index.html'); assert.strictEqual(daCtx.aemPathname, '/'); }); }); @@ -94,7 +94,7 @@ describe('DA context', () => { }); it('should return the correct path names', () => { - assert.strictEqual(daCtx.pathname, '/index'); + assert.strictEqual(daCtx.sourcePath, '/index.html'); assert.strictEqual(daCtx.aemPathname, '/'); }); }); @@ -105,7 +105,7 @@ describe('DA context', () => { }); it('should return the correct path names', () => { - assert.strictEqual(daCtx.pathname, '/sub-folder/index'); + assert.strictEqual(daCtx.sourcePath, '/sub-folder/index.html'); assert.strictEqual(daCtx.aemPathname, '/sub-folder/'); }); }); @@ -115,11 +115,10 @@ describe('DA context', () => { daCtx = getDaCtx(reqs.nonHtmlFile); }); - it('should return the correct pathname with extension', () => { - assert.strictEqual(daCtx.pathname, '/folder/content.json'); + it('should return the correct path names with extension', () => { + assert.strictEqual(daCtx.sourcePath, '/folder/content.json'); assert.strictEqual(daCtx.aemPathname, '/folder/content.json'); assert.strictEqual(daCtx.ext, 'json'); - assert.strictEqual(daCtx.name, 'content'); }); }); @@ -128,11 +127,10 @@ describe('DA context', () => { daCtx = getDaCtx(reqs.plainFile); }); - it('should return the correct pathname with extension', () => { - assert.strictEqual(daCtx.pathname, '/folder/content.plain.html'); + it('should return the correct path names with extension', () => { + assert.strictEqual(daCtx.sourcePath, '/folder/content.plain.html'); assert.strictEqual(daCtx.aemPathname, '/folder/content.plain.html'); assert.strictEqual(daCtx.ext, 'html'); - assert.strictEqual(daCtx.name, 'content.plain'); }); }); From 458bba72c0d2bb6a1a47e98cf351ad31ce6d43f5 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 15:04:14 +0200 Subject: [PATCH 03/48] test: pin the post gate against charset and binary types 4 red: text/html with a charset param is refused, octet-stream and text/plain are allowed through onto the key GET reads. the two case rows pass in node because undici normalizes File.type; they are workerd regression guards. --- test/routes/da-admin.test.js | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index 9bd9adbf..8d100894 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -375,4 +375,44 @@ describe('daSourcePost', () => { assert.ok(res instanceof Response); assert.deepStrictEqual(fetched, []); }); + + ['text/html; charset=utf-8', 'text/html;charset=UTF-8', 'TEXT/HTML', 'text/HTML '].forEach((type) => { + it(`writes an HTML File declared as "${type}"`, async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'page.html', { type }); + const req = formReq('https://main--site--org.ue.da.live/page', html); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 200); + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); + }); + + ['application/octet-stream', 'text/plain', 'image/svg+xml', 'application/pdf'].forEach((type) => { + it(`refuses a File declared as "${type}"`, async () => { + const { env, fetched } = recorder(); + const file = new File([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], 'logo.png', { type }); + const req = formReq('https://main--site--org.ue.da.live/media/logo.png', file); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 415); + assert.deepStrictEqual(fetched, []); + }); + }); + + it('writes a File with no declared type', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'page.html'); + const req = formReq('https://main--site--org.ue.da.live/page', html); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 200); + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); }); From acfa57e820b68e2e88860fe76376f1ef64b506e9 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 15:07:14 +0200 Subject: [PATCH 04/48] fix: accept only text/html on a file post, ignoring charset and case workerd preserves a part's declared type verbatim and reports '' when the client declared none, so the check normalizes before comparing and no longer allows octet-stream or text/plain onto the key GET reads. --- src/routes/da-admin.js | 19 +++++++++++++++---- test/routes/da-admin.test.js | 26 ++++++++++++++++---------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index d2d9db87..fce4f77f 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -28,9 +28,20 @@ import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, UNAUTHORIZED_HTML import { getSiteConfig } from '../storage/config.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; -// file content types accepted on an HTML POST. formData() reports octet-stream or -// text/plain for a part whose type the client left unset, so both stay allowed. -const HTML_POST_TYPES = ['text/html', 'text/plain', 'application/octet-stream']; +const HTML_POST_TYPE = 'text/html'; + +/** + * Whether a multipart part's type may go through the HTML serializer. Anything else + * would be read with `.text()` and written back mangled, so it is refused. + * An empty type is what workerd reports when the client declared none, and is allowed. + * + * @param {string} type The part's content type + * @returns {boolean} + */ +export function isHtmlPostType(type) { + if (!type) return true; + return type.split(';')[0].trim().toLowerCase() === HTML_POST_TYPE; +} async function getFileBody(data) { const text = await data.text(); @@ -189,7 +200,7 @@ export async function daSourcePost({ req, env, daCtx }) { const obj = await putHelper(req, env, daCtx); if (obj && obj.data) { const isFile = obj.data instanceof File; - if (isFile && obj.data.type && !HTML_POST_TYPES.includes(obj.data.type)) { + if (isFile && !isHtmlPostType(obj.data.type)) { return get415(); } const { body: bodyHtml } = isFile diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index 8d100894..e6a4f9a7 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -16,7 +16,7 @@ import esmock from 'esmock'; import reqs from '../mocks/req.js'; const { getDaCtx } = await import('../../src/utils/daCtx.js'); -const { daSourceHead, daSourcePost } = await import('../../src/routes/da-admin.js'); +const { daSourceHead, daSourcePost, isHtmlPostType } = await import('../../src/routes/da-admin.js'); const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer t' } }); @@ -403,16 +403,22 @@ describe('daSourcePost', () => { assert.deepStrictEqual(fetched, []); }); }); +}); - it('writes a File with no declared type', async () => { - const { env, fetched } = recorder(); - const html = new File(['hello'], 'page.html'); - const req = formReq('https://main--site--org.ue.da.live/page', html); - const daCtx = getDaCtx(req); - - const res = await daSourcePost({ req, env, daCtx }); +// workerd and undici disagree on File.type for a multipart part, so the rule is +// asserted directly. measured in workerd 4.118.0: an absent part Content-Type +// reports '', and a declared one is preserved verbatim including its case and +// parameters. undici substitutes application/octet-stream and lowercases. +describe('isHtmlPostType', () => { + ['', 'text/html', 'text/html; charset=utf-8', 'text/html;charset=UTF-8', 'TEXT/HTML', 'text/HTML '].forEach((type) => { + it(`accepts ${JSON.stringify(type)}`, () => { + assert.strictEqual(isHtmlPostType(type), true); + }); + }); - assert.strictEqual(res.status, 200); - assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + ['application/octet-stream', 'text/plain', 'image/png', 'image/svg+xml', 'application/pdf', 'application/json'].forEach((type) => { + it(`rejects ${JSON.stringify(type)}`, () => { + assert.strictEqual(isHtmlPostType(type), false); + }); }); }); From 9d5d4d02537b61097dce8e64c1ac479b88bde292 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 19:04:17 +0200 Subject: [PATCH 05/48] test: refuse a post that does not address an html document 3 red. the html serializer rewrites any body, so posting to /Media/Logo.PNG, a string part, or /sheet.json currently writes html onto the canonical key that GET reads. on main those went to a doubled key nothing reads. --- test/routes/da-admin.test.js | 39 +++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index e6a4f9a7..d5bb91a4 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -322,16 +322,49 @@ describe('source URLs', () => { assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); }); +}); - it('POST /Media/Logo.PNG writes /media/logo.png', async () => { +// the HTML serializer would rewrite whatever it is given, so a POST is refused +// unless it addresses an HTML document. sourcePath ends `.html` iff ext is html, +// so this covers every non-HTML target. +describe('daSourcePost to a non-HTML path', () => { + it('refuses an HTML File and does not write', async () => { const { env, fetched } = recorder(); const html = new File(['hello'], 'logo.html', { type: 'text/html' }); const req = formReq('https://main--site--org.ue.da.live/Media/Logo.PNG', html); const daCtx = getDaCtx(req); - await daSourcePost({ req, env, daCtx }); + const res = await daSourcePost({ req, env, daCtx }); - assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/media/logo.png']); + assert.strictEqual(res.status, 415); + assert.deepStrictEqual(fetched, []); + }); + + // an untyped part is the case the part-type check cannot cover in node, since + // undici substitutes application/octet-stream where workerd reports ''. The path + // check catches it either way, so this is asserted on a string part, which + // carries no type in either runtime. + it('refuses a string part, which carries no type at all, and does not write', async () => { + const { env, fetched } = recorder(); + const req = formReq('https://main--site--org.ue.da.live/media/logo.png', 'hello'); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 415); + assert.deepStrictEqual(fetched, []); + }); + + it('refuses a POST to a json path', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'sheet.html', { type: 'text/html' }); + const req = formReq('https://main--site--org.ue.da.live/sheet.json', html); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 415); + assert.deepStrictEqual(fetched, []); }); }); From 532b67a506c676f1fbb4c25d67cdc6332b1db1c3 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 19:05:17 +0200 Subject: [PATCH 06/48] fix: refuse a post that does not address an html document ext is html iff sourcePath ends .html, so this covers every non-html target and refuses nothing the universal editor produces. main sent those writes to a doubled key; sourcePath sends them to the key GET reads. --- src/routes/da-admin.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index fce4f77f..5a8105ca 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -194,9 +194,16 @@ export async function daSourceHead({ env, daCtx }) { export async function daSourcePost({ req, env, daCtx }) { const { - org, site, sourcePath, authToken, + org, site, sourcePath, ext, authToken, } = daCtx; + // the body is rewritten as HTML below, so anything but an HTML document would be + // written back mangled onto the key GET reads + if (ext !== 'html') { + console.log(`415 POST ${sourcePath}, not an HTML document`); + return get415(); + } + const obj = await putHelper(req, env, daCtx); if (obj && obj.data) { const isFile = obj.data instanceof File; From 48cb6328a2180573364425d710ac3900894eced4 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 19:06:52 +0200 Subject: [PATCH 07/48] test: pin the 415 contract and drop the duplicated type matrices the non-form fallthrough asserted only instanceof Response, so a 200 passed. the part-type tests targeted a non-html path, where the path check now answers first. adds the urlencoded string-part path, which had no test. --- test/routes/da-admin.test.js | 62 +++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index d5bb91a4..777c3e31 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -369,11 +369,13 @@ describe('daSourcePost to a non-HTML path', () => { }); describe('daSourcePost', () => { + // on an HTML path, so the path check does not answer first and this exercises + // the part-type check it('refuses a binary File with 415 and does not write', async () => { const { env, fetched } = recorder(); const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]); const png = new File([bytes], 'logo.png', { type: 'image/png' }); - const req = formReq('https://main--site--org.ue.da.live/media/logo.png', png); + const req = formReq('https://main--site--org.ue.da.live/page', png); const daCtx = getDaCtx(req); const res = await daSourcePost({ req, env, daCtx }); @@ -394,7 +396,22 @@ describe('daSourcePost', () => { assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); }); - it('returns a response when the content type is not a form type', async () => { + it('writes a string part', async () => { + const { env, fetched } = recorder(); + const req = new Request('https://main--site--org.ue.da.live/page', { + method: 'POST', + body: new URLSearchParams({ data: 'hello' }), + headers: { Authorization: 'Bearer t' }, + }); + const daCtx = getDaCtx(req); + + const res = await daSourcePost({ req, env, daCtx }); + + assert.strictEqual(res.status, 200); + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); + }); + + it('refuses 415 with an empty body when the request content type is not a form type', async () => { const { env, fetched } = recorder(); const req = new Request('https://main--site--org.ue.da.live/page', { method: 'POST', @@ -405,36 +422,35 @@ describe('daSourcePost', () => { const res = await daSourcePost({ req, env, daCtx }); - assert.ok(res instanceof Response); + assert.strictEqual(res.status, 415); + assert.strictEqual(await res.text(), ''); assert.deepStrictEqual(fetched, []); }); - ['text/html; charset=utf-8', 'text/html;charset=UTF-8', 'TEXT/HTML', 'text/HTML '].forEach((type) => { - it(`writes an HTML File declared as "${type}"`, async () => { - const { env, fetched } = recorder(); - const html = new File(['hello'], 'page.html', { type }); - const req = formReq('https://main--site--org.ue.da.live/page', html); - const daCtx = getDaCtx(req); + // the full normalization matrix is under isHtmlPostType; these two prove it is + // wired into the route. the charset form is what da-admin itself sends back. + it('writes an HTML File declared with a charset', async () => { + const { env, fetched } = recorder(); + const html = new File(['hello'], 'page.html', { type: 'text/html; charset=utf-8' }); + const req = formReq('https://main--site--org.ue.da.live/page', html); + const daCtx = getDaCtx(req); - const res = await daSourcePost({ req, env, daCtx }); + const res = await daSourcePost({ req, env, daCtx }); - assert.strictEqual(res.status, 200); - assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); - }); + assert.strictEqual(res.status, 200); + assert.deepStrictEqual(fetched, ['https://admin.da.live/source/org/site/page.html']); }); - ['application/octet-stream', 'text/plain', 'image/svg+xml', 'application/pdf'].forEach((type) => { - it(`refuses a File declared as "${type}"`, async () => { - const { env, fetched } = recorder(); - const file = new File([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], 'logo.png', { type }); - const req = formReq('https://main--site--org.ue.da.live/media/logo.png', file); - const daCtx = getDaCtx(req); + it('refuses a File declared as application/octet-stream on an HTML path', async () => { + const { env, fetched } = recorder(); + const file = new File([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], 'logo.png', { type: 'application/octet-stream' }); + const req = formReq('https://main--site--org.ue.da.live/page', file); + const daCtx = getDaCtx(req); - const res = await daSourcePost({ req, env, daCtx }); + const res = await daSourcePost({ req, env, daCtx }); - assert.strictEqual(res.status, 415); - assert.deepStrictEqual(fetched, []); - }); + assert.strictEqual(res.status, 415); + assert.deepStrictEqual(fetched, []); }); }); From b66a96bea52c78a46ac1249d382f14eedbf6dd6d Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 20:19:14 +0200 Subject: [PATCH 08/48] refactor: read ue-service with searchParams.get drops the intermediate object. on a duplicated param this takes the first value where fromEntries took the last; only 'local' is acted on, at ue/scaffold.js:36. --- src/utils/daCtx.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils/daCtx.js b/src/utils/daCtx.js index b2cf1338..42076717 100644 --- a/src/utils/daCtx.js +++ b/src/utils/daCtx.js @@ -110,10 +110,8 @@ export function getDaCtx(req) { daCtx.aemPathname = path.endsWith('/index') ? path.substring(0, path.length - 5) : path; daCtx.sourcePath = `/${[...pathParts, dotted ? filename : `${filename}.html`].join('/')}`; - const query = Object.fromEntries(searchParams.entries()); - if (typeof query['ue-service'] === 'string') { - daCtx.ueService = query['ue-service']; - } + const ueService = searchParams.get('ue-service'); + if (ueService !== null) daCtx.ueService = ueService; daCtx.authToken = getAuthToken(req); daCtx.siteToken = getSiteToken(req); From b0cfa3c28d3b3934578f2e6a94ddd041d3585ab3 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 20:19:14 +0200 Subject: [PATCH 09/48] docs: drop the isHtmlPostType jsdoc the workerd empty-type behaviour it described is in the acfa57e commit message. --- src/routes/da-admin.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index 5a8105ca..eec548f4 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -30,14 +30,6 @@ import { restoreAbsoluteImages } from '../render/rewrite-images.js'; const HTML_POST_TYPE = 'text/html'; -/** - * Whether a multipart part's type may go through the HTML serializer. Anything else - * would be read with `.text()` and written back mangled, so it is refused. - * An empty type is what workerd reports when the client declared none, and is allowed. - * - * @param {string} type The part's content type - * @returns {boolean} - */ export function isHtmlPostType(type) { if (!type) return true; return type.split(';')[0].trim().toLowerCase() === HTML_POST_TYPE; From 1af5bcc74190f1ee0dd881ed5268dd93b8acd40c Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Sat, 1 Aug 2026 22:16:47 +0200 Subject: [PATCH 10/48] refactor: build the da-admin source url in one place the four call sites had the same construction in three spellings. org, site and sourcePath drop out of three destructures as a result. --- src/routes/da-admin.js | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index eec548f4..65f2508e 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -35,6 +35,10 @@ export function isHtmlPostType(type) { return type.split(';')[0].trim().toLowerCase() === HTML_POST_TYPE; } +function getSourceUrl(env, { org, site, sourcePath }) { + return new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); +} + async function getFileBody(data) { const text = await data.text(); return { body: text, type: data.type }; @@ -77,9 +81,7 @@ async function getPageTemplate(env, daCtx, aemCtx) { } export async function daSourceGet({ req, env, daCtx }) { - const { - org, site, sourcePath, ext, authToken, - } = daCtx; + const { ext, authToken } = daCtx; // check if Authorization header is present if (!authToken) { @@ -99,7 +101,7 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing - const adminUrl = new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); + const adminUrl = getSourceUrl(env, daCtx); console.log(`-> ${adminUrl.toString()}`); const response = await env.daadmin.fetch(adminUrl, { method: 'GET', headers }); console.log(`<- ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); @@ -119,10 +121,7 @@ export async function daSourceGet({ req, env, daCtx }) { } // get the content from DA admin - const adminUrl = new URL( - `/source/${org}/${site}${sourcePath}`, - env.DA_ADMIN, - ); + const adminUrl = getSourceUrl(env, daCtx); // eslint-disable-next-line no-param-reassign req = new Request(adminUrl, { @@ -166,9 +165,7 @@ export async function daSourceGet({ req, env, daCtx }) { } export async function daSourceHead({ env, daCtx }) { - const { - org, site, sourcePath, authToken, - } = daCtx; + const { authToken } = daCtx; if (!authToken) { return head401(); @@ -177,7 +174,7 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const adminUrl = new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); + const adminUrl = getSourceUrl(env, daCtx); console.log(`-> HEAD ${adminUrl.toString()}`); const response = await env.daadmin.fetch(adminUrl, { method: 'HEAD', headers }); console.log(`<- HEAD ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); @@ -185,9 +182,7 @@ export async function daSourceHead({ env, daCtx }) { } export async function daSourcePost({ req, env, daCtx }) { - const { - org, site, sourcePath, ext, authToken, - } = daCtx; + const { sourcePath, ext, authToken } = daCtx; // the body is rewritten as HTML below, so anything but an HTML document would be // written back mangled onto the key GET reads @@ -224,10 +219,7 @@ export async function daSourcePost({ req, env, daCtx }) { const data = new Blob([bodyContent], { type: 'text/html' }); body.set('data', data); const headers = { Authorization: authToken }; - const adminUrl = new URL( - `/source/${org}/${site}${sourcePath}`, - env.DA_ADMIN, - ); + const adminUrl = getSourceUrl(env, daCtx); // eslint-disable-next-line no-param-reassign req = new Request(adminUrl, { method: 'POST', From be3a892549b4436e9513be7e4b65206bebb1be00 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:05:10 +0200 Subject: [PATCH 11/48] test: pin how the content source is resolved the sidekick config names the store and 404s when config resolution failed, so it can answer "unknown" where /ping reports legacy. --- test/storage/content-source.test.js | 219 ++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 test/storage/content-source.test.js diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js new file mode 100644 index 00000000..cb052ad8 --- /dev/null +++ b/test/storage/content-source.test.js @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; + +const { default: resolveContentSource } = await import('../../src/storage/content-source.js'); + +const env = { HLX_ADMIN: 'https://admin.hlx.page' }; + +const daCtx = (over = {}) => ({ + org: 'org', site: 'site', ref: 'main', authToken: 'Bearer t', ...over, +}); + +let calls; + +const stubFetch = (respond) => { + calls = []; + globalThis.fetch = async (input, init) => { + calls.push({ url: input.toString(), init }); + return respond(input.toString(), init); + }; +}; + +const sidekick = (contentSourceUrl) => new Response( + JSON.stringify({ contentSourceUrl, contentSourceType: 'markup' }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, +); + +const legacyBody = () => sidekick('https://content.da.live/org/site/'); + +describe('resolveContentSource', () => { + afterEach(() => { + delete globalThis.fetch; + }); + + describe('the request it makes', () => { + it('asks the sidekick config for org, site and ref', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx({ ref: 'branch' })); + + assert.strictEqual(calls.length, 1); + assert.strictEqual( + calls[0].url, + 'https://admin.hlx.page/sidekick/org/site/branch/config.json', + ); + }); + + it('passes the author token on, so a private site resolves', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx()); + + assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), 'Bearer t'); + }); + + it('asks anyway when there is no author token', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx({ authToken: undefined })); + + assert.strictEqual(calls.length, 1); + assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), null); + }); + }); + + describe('when the content source is on api.aem.live', () => { + it('answers sourcebus', async () => { + stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'sourcebus'); + }); + + it('carries the base url from the config, rather than rebuilding it', async () => { + stubFetch(() => sidekick('https://api.aem.live/other/sites/elsewhere/source')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.base, 'https://api.aem.live/other/sites/elsewhere/source'); + }); + + it('drops a trailing slash on the base, so paths do not double up', async () => { + stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source/')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.base, 'https://api.aem.live/org/sites/site/source'); + }); + }); + + describe('when the content source is on content.da.live', () => { + it('answers legacy', async () => { + stubFetch(legacyBody); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'legacy'); + }); + }); + + describe('when the answer is not one of the two stores', () => { + // this worker only serves DA-backed sites, so a google or onedrive mount is not + // something either store can answer for and must not be guessed at + it('answers unknown for a source url it does not recognise', async () => { + stubFetch(() => sidekick('https://drive.google.com/drive/folders/abc')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown when contentSourceUrl is missing from the body', async () => { + stubFetch(() => new Response(JSON.stringify({ project: 'site' }), { status: 200 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown for a host that only starts like api.aem.live', async () => { + stubFetch(() => sidekick('https://api.aem.live.evil.example/org/sites/site/source')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + }); + + describe('when the question could not be answered', () => { + // a 404 is what helix-admin returns when config resolution produced nothing + // (src/sidekick/handler.js: `if (config) { ... } return { status: 404 }`), so it + // means "we do not know", not "legacy" + it('answers unknown on a 404', async () => { + stubFetch(() => new Response('', { status: 404 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown on a 5xx', async () => { + stubFetch(() => new Response('', { status: 503 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown on a 401', async () => { + stubFetch(() => new Response('', { status: 401 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown when the body is not json', async () => { + stubFetch(() => new Response('gateway', { status: 200 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown when the fetch throws', async () => { + stubFetch(() => { + throw new Error('boom'); + }); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('says why it could not answer', async () => { + stubFetch(() => new Response('', { status: 404 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.match(source.reason, /404/); + }); + }); + + describe('when there is no site to ask about', () => { + it('answers unknown without making a request', async () => { + stubFetch(legacyBody); + + const source = await resolveContentSource(env, daCtx({ org: undefined, site: undefined })); + + assert.strictEqual(source.kind, 'unknown'); + assert.strictEqual(calls.length, 0); + }); + }); + + describe('the admin host', () => { + it('comes from env, so stage can point elsewhere', async () => { + stubFetch(legacyBody); + + await resolveContentSource({ HLX_ADMIN: 'https://admin.stage.example' }, daCtx()); + + assert.strictEqual( + calls[0].url, + 'https://admin.stage.example/sidekick/org/site/main/config.json', + ); + }); + }); +}); From e53df66089f9831ec82b040abfd0e32dd0e9c828 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:06:01 +0200 Subject: [PATCH 12/48] feat: resolve the content source from the sidekick config answers sourcebus with the store base url, legacy, or unknown. a config that could not be resolved lands on unknown rather than being reported as legacy. --- src/storage/content-source.js | 84 +++++++++++++++++++++++++++++++++++ wrangler.toml | 6 +-- 2 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 src/storage/content-source.js diff --git a/src/storage/content-source.js b/src/storage/content-source.js new file mode 100644 index 00000000..4a96abd4 --- /dev/null +++ b/src/storage/content-source.js @@ -0,0 +1,84 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +const SOURCE_BUS_PREFIX = 'https://api.aem.live/'; +const LEGACY_PREFIX = 'https://content.da.live/'; +const TIMEOUT_MS = 5 * 1000; + +export const SOURCE_BUS = 'sourcebus'; +export const LEGACY = 'legacy'; +export const UNKNOWN = 'unknown'; + +function unknown(org, site, reason) { + console.warn(`[source] ${org}/${site} unknown: ${reason}`); + return { kind: UNKNOWN, reason }; +} + +/** + * Asks admin.hlx.page which store holds a site's content. + * + * The sidekick config returns the resolved content source url in its body, and answers 404 when + * config resolution produced nothing (`if (config) { ... } return { status: 404 }` in + * helix-admin src/sidekick/handler.js). So a failure to resolve is reported as a failure. The + * `/ping` header cannot do that: it is absent both for a legacy site and for a source-bus site + * whose config could not be read, which is the case that reads past a page and writes over it. + * + * @param {Object} env worker env, `HLX_ADMIN` is the admin host + * @param {Object} daCtx + * @returns {Promise<{kind: string, base?: string, reason?: string}>} `sourcebus` with the store + * base url, `legacy`, or `unknown` with the reason it could not be answered + */ +export default async function resolveContentSource(env, daCtx) { + const { + org, site, ref, authToken, + } = daCtx; + + // an unparseable hostname leaves org, site and ref all undefined together, and there is no + // site to ask about + if (!org || !site) { + return unknown(org, site, 'no org or site in the request'); + } + + const url = new URL(`/sidekick/${org}/${site}/${ref}/config.json`, env.HLX_ADMIN); + const headers = new Headers(); + if (authToken) headers.set('Authorization', authToken); + + let response; + try { + response = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS) }); + } catch (e) { + return unknown(org, site, `${url} failed with ${e.name}: ${e.message}`); + } + + if (response.status !== 200) { + return unknown(org, site, `${url} answered ${response.status}`); + } + + let config; + try { + config = await response.json(); + } catch (e) { + return unknown(org, site, `${url} did not answer json: ${e.message}`); + } + + const sourceUrl = config?.contentSourceUrl; + if (typeof sourceUrl !== 'string') { + return unknown(org, site, `${url} named no content source`); + } + if (sourceUrl.startsWith(SOURCE_BUS_PREFIX)) { + return { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; + } + if (sourceUrl.startsWith(LEGACY_PREFIX)) { + return { kind: LEGACY }; + } + return unknown(org, site, `content source ${sourceUrl} is neither store`); +} diff --git a/wrangler.toml b/wrangler.toml index 3d7fbdd7..73dba068 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,18 +2,18 @@ name = "da-ue" main = "src/index.js" compatibility_date = "2023-11-21" -vars = { UE_HOST = "ue.da.live", DA_ADMIN = "https://admin.da.live" } +vars = { UE_HOST = "ue.da.live", DA_ADMIN = "https://admin.da.live", HLX_ADMIN = "https://admin.hlx.page" } services = [{ binding = "daadmin", service = "da-admin" }] [dev] port = 4712 [env.dev] -vars = { ENVIRONMENT = "dev", UE_HOST = "localhost:4712", UE_SERVICE = "https://localhost:8000", DA_ADMIN = "https://admin.da.live" } +vars = { ENVIRONMENT = "dev", UE_HOST = "localhost:4712", UE_SERVICE = "https://localhost:8000", DA_ADMIN = "https://admin.da.live", HLX_ADMIN = "https://admin.hlx.page" } services = [{ binding = "daadmin", service = "da-admin-local" }] [env.stage] -vars = { ENVIRONMENT = "stage", UE_HOST = "stage-ue.da.live", UE_SERVICE = "https://universal-editor-service-dev.adobe.io", DA_ADMIN = "https://admin.da.live" } +vars = { ENVIRONMENT = "stage", UE_HOST = "stage-ue.da.live", UE_SERVICE = "https://universal-editor-service-dev.adobe.io", DA_ADMIN = "https://admin.da.live", HLX_ADMIN = "https://admin.hlx.page" } services = [{ binding = "daadmin", service = "da-admin" }] [env.stage.observability] From 065e20146b002eb3e1ad17ac9f2083a668eee552 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:07:12 +0200 Subject: [PATCH 13/48] test: pin each store's url shape and write body the source bus keeps directory and extension case and reads a raw body; da-admin lowercases the whole path and reads a data form part. --- test/storage/store.test.js | 175 +++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 test/storage/store.test.js diff --git a/test/storage/store.test.js b/test/storage/store.test.js new file mode 100644 index 00000000..efd5fd9c --- /dev/null +++ b/test/storage/store.test.js @@ -0,0 +1,175 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import { getDaCtx } from '../../src/utils/daCtx.js'; +import { LEGACY, SOURCE_BUS } from '../../src/storage/content-source.js'; + +const { default: getStore, sourceBusPath } = await import('../../src/storage/store.js'); + +const env = { DA_ADMIN: 'https://admin.da.live' }; +const ctxFor = (url) => getDaCtx(new Request(url, { headers: { Authorization: 'Bearer t' } })); +const legacy = { kind: LEGACY }; +const bus = { kind: SOURCE_BUS, base: 'https://api.aem.live/org/sites/site/source' }; + +describe('sourceBusPath', () => { + // helix-api-service sanitizes only the basename: computePaths pops the filename, runs + // sanitizeName on it and recombines the directory segments untouched. Verified live on + // 2026-08-03 by uploading /Media/CaseProbe.PNG and fetching six spellings back: only + // /Media/CaseProbe.PNG and /Media/caseprobe.PNG answered 200. + const cases = [ + ['/folder/content', '/folder/content.html', 'appends .html when the request had no extension'], + ['/', '/index.html', 'names the root document index.html'], + ['/sub-folder/', '/sub-folder/index.html', 'names a directory index'], + ['/Media/Holiday.PNG', '/Media/holiday.PNG', 'lowercases the stem, keeps directory and extension case'], + ['/A/B/c.JSON', '/A/B/c.JSON', 'keeps every directory segment as requested'], + ['/Sub-Folder/', '/Sub-Folder/index.html', 'keeps directory case on a directory index'], + ['/folder/Content', '/folder/content.html', 'lowercases a stem that had no extension'], + ['/folder/content.plain.html', '/folder/content.plain.html', 'treats only the last dot as the extension'], + ]; + + cases.forEach(([path, expected, what]) => { + it(what, () => { + assert.strictEqual(sourceBusPath(ctxFor(`https://main--site--org.ue.da.live${path}`)), expected); + }); + }); + + it('differs from daCtx.sourcePath, which da-admin wants lowercased throughout', () => { + const daCtx = ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'); + + assert.strictEqual(daCtx.sourcePath, '/media/holiday.png'); + assert.strictEqual(sourceBusPath(daCtx), '/Media/holiday.PNG'); + }); +}); + +describe('getStore', () => { + describe('the url it reads and writes', () => { + it('builds a legacy url under DA_ADMIN from the lowercased source path', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Folder/Doc'), legacy); + + assert.strictEqual(store.url.toString(), 'https://admin.da.live/source/org/site/folder/doc.html'); + }); + + it('builds a source-bus url on the base the config named', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/folder/doc'), bus); + + assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/folder/doc.html'); + }); + + it('keeps the case the source bus stored a file under', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'), bus); + + assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/Media/holiday.PNG'); + }); + + it('takes the base verbatim, so a config naming another org is followed', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), { + kind: SOURCE_BUS, + base: 'https://api.aem.live/shared/sites/library/source', + }); + + assert.strictEqual(store.url.toString(), 'https://api.aem.live/shared/sites/library/source/doc.html'); + }); + }); + + describe('how it reaches the store', () => { + it('sends a legacy request over the daadmin service binding', async () => { + const seen = []; + const bound = { ...env, daadmin: { fetch: async (i) => { seen.push(i); return new Response(''); } } }; + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + + await store.fetch(store.url, { method: 'GET' }); + + assert.strictEqual(seen.length, 1); + }); + + it('sends a source-bus request over the public network', async () => { + const seen = []; + globalThis.fetch = async (i) => { seen.push(i); return new Response(''); }; + const bound = { ...env, daadmin: { fetch: async () => assert.fail('used the binding') } }; + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + + await store.fetch(store.url, { method: 'GET' }); + + delete globalThis.fetch; + assert.strictEqual(seen.length, 1); + }); + }); + + describe('the write body each store parses', () => { + // helix-api-service parses no form data anywhere: getValidPayload reads the raw buffer and + // types it from the path extension. Sending da-admin's multipart envelope stores the + // boundary lines as the document text and answers 201, so this is not a cosmetic difference. + it('sends the source bus the document as the raw body', async () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + + const init = store.writeInit('

hi

', 'Bearer t'); + const body = await new Request('https://example.test', { method: 'POST', ...init }).text(); + + assert.strictEqual(body, '

hi

'); + }); + + it('types the source-bus write as text/html', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + + const init = store.writeInit('', 'Bearer t'); + + assert.strictEqual(new Headers(init.headers).get('Content-Type'), 'text/html'); + }); + + it('sends da-admin the document as a data form part', async () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + + const init = store.writeInit('

hi

', 'Bearer t'); + const form = await new Request('https://example.test', { method: 'POST', ...init }).formData(); + + assert.strictEqual(await form.get('data').text(), '

hi

'); + assert.strictEqual(form.get('data').type, 'text/html'); + }); + + it('never wraps a source-bus write in a multipart envelope', async () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + + const init = store.writeInit('', 'Bearer t'); + const body = await new Request('https://example.test', { method: 'POST', ...init }).text(); + + assert.ok(!body.includes('Content-Disposition'), `envelope leaked into the body: ${body}`); + assert.ok(!body.includes('form-data'), `envelope leaked into the body: ${body}`); + }); + + it('authorizes both writes with the caller token', () => { + const b = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + const l = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + + assert.strictEqual(new Headers(b.writeInit('', 'Bearer t').headers).get('Authorization'), 'Bearer t'); + assert.strictEqual(new Headers(l.writeInit('', 'Bearer t').headers).get('Authorization'), 'Bearer t'); + }); + + it('adds a precondition to the source-bus write when one is given', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + + const init = store.writeInit('', 'Bearer t', { 'If-Match': '"abc"' }); + + assert.strictEqual(new Headers(init.headers).get('If-Match'), '"abc"'); + }); + + it('sends no precondition when none is given', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + + const init = store.writeInit('', 'Bearer t'); + + assert.strictEqual(new Headers(init.headers).get('If-Match'), null); + assert.strictEqual(new Headers(init.headers).get('If-None-Match'), null); + }); + }); +}); From 6471fa08dbe832867cf40c1fab5921952afba57f Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:07:39 +0200 Subject: [PATCH 14/48] feat: add the store adapter and the source-bus path case each store builds its own url and its own write body, since neither parses the other's shape. --- src/storage/store.js | 72 ++++++++++++++++++++++++++++++++++++++ test/storage/store.test.js | 11 ++++-- 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 src/storage/store.js diff --git a/src/storage/store.js b/src/storage/store.js new file mode 100644 index 00000000..93892b39 --- /dev/null +++ b/src/storage/store.js @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { SOURCE_BUS } from './content-source.js'; + +/** + * Restores the case the source bus stores a file under. + * + * helix-api-service sanitizes the basename and nothing else: `computePaths` pops the filename, + * runs `sanitizeName` on it and recombines the directory segments untouched. The extension comes + * back verbatim too. `daCtx.sourcePath` lowercases the whole path, which is what da-admin wants + * and what the source bus 404s on. + * + * @param {Object} daCtx + * @returns {string} the store path, directory and extension in the case they were requested + */ +export function sourceBusPath({ path, sourcePath }) { + const dirEnd = path.lastIndexOf('/'); + const base = sourcePath.slice(sourcePath.lastIndexOf('/') + 1); + const baseDot = base.lastIndexOf('.'); + const requestedDot = path.lastIndexOf('.'); + // an extension the request carried keeps its case; the `.html` we appended does not have one + const ext = requestedDot > dirEnd ? path.slice(requestedDot) : base.slice(baseDot); + return `${path.slice(0, dirEnd)}/${base.slice(0, baseDot)}${ext}`; +} + +/** + * Picks the store for a request, the way to reach it, and the shape it takes a write in. + * + * da-admin answers over a service binding and the source bus over the public network, so one + * fetch cannot serve both. They also differ on the write body: helix-api-service reads the raw + * request body and types it from the path extension, parsing no form data anywhere, while + * da-admin takes the document as a `data` form part. Handing either the other's shape stores + * something other than the document and answers 201. + * + * @param {Object} env worker env + * @param {Object} daCtx + * @param {{kind: string, base?: string}} source the resolved content source + */ +export default function getStore(env, daCtx, source) { + const { org, site, sourcePath } = daCtx; + + if (source.kind === SOURCE_BUS) { + return { + url: new URL(`${source.base}${sourceBusPath(daCtx)}`), + fetch: (input, init) => fetch(input, init), + writeInit: (html, authToken, condition) => ({ + method: 'POST', + body: html, + headers: { Authorization: authToken, 'Content-Type': 'text/html', ...condition }, + }), + }; + } + + return { + url: new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN), + fetch: (input, init) => env.daadmin.fetch(input, init), + writeInit: (html, authToken, condition) => { + const body = new FormData(); + body.set('data', new Blob([html], { type: 'text/html' })); + return { method: 'POST', body, headers: { Authorization: authToken, ...condition } }; + }, + }; +} diff --git a/test/storage/store.test.js b/test/storage/store.test.js index efd5fd9c..8fbaa8f8 100644 --- a/test/storage/store.test.js +++ b/test/storage/store.test.js @@ -85,7 +85,11 @@ describe('getStore', () => { describe('how it reaches the store', () => { it('sends a legacy request over the daadmin service binding', async () => { const seen = []; - const bound = { ...env, daadmin: { fetch: async (i) => { seen.push(i); return new Response(''); } } }; + const record = async (input) => { + seen.push(input); + return new Response(''); + }; + const bound = { ...env, daadmin: { fetch: record } }; const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); await store.fetch(store.url, { method: 'GET' }); @@ -95,7 +99,10 @@ describe('getStore', () => { it('sends a source-bus request over the public network', async () => { const seen = []; - globalThis.fetch = async (i) => { seen.push(i); return new Response(''); }; + globalThis.fetch = async (input) => { + seen.push(input); + return new Response(''); + }; const bound = { ...env, daadmin: { fetch: async () => assert.fail('used the binding') } }; const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), bus); From f8db1d1d1cb27b3305ef5588bdae97e5beb2e523 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:08:48 +0200 Subject: [PATCH 15/48] test: pin the stamp that links a read to its write a source-bus read carries its etag so the write is conditional; anything a stamp cannot be trusted to say is not trusted. --- test/utils/source-stamp.test.js | 131 ++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 test/utils/source-stamp.test.js diff --git a/test/utils/source-stamp.test.js b/test/utils/source-stamp.test.js new file mode 100644 index 00000000..b7e79a0b --- /dev/null +++ b/test/utils/source-stamp.test.js @@ -0,0 +1,131 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import { LEGACY, SOURCE_BUS } from '../../src/storage/content-source.js'; + +const { + SOURCE_STAMP_PARAM, formatSourceStamp, parseSourceStamp, +} = await import('../../src/utils/source-stamp.js'); + +const bus = { kind: SOURCE_BUS, base: 'https://api.aem.live/org/sites/site/source' }; +const legacy = { kind: LEGACY }; + +describe('formatSourceStamp', () => { + it('names the param the connection uri carries', () => { + assert.strictEqual(SOURCE_STAMP_PARAM, 'ab-src'); + }); + + it('stamps a source-bus read with the etag it read', () => { + assert.strictEqual(formatSourceStamp(bus, '"9e8311043aab12b1"'), 'sb.9e8311043aab12b1'); + }); + + it('strips the quotes, so the stamp needs no url encoding', () => { + assert.doesNotMatch(formatSourceStamp(bus, '"abc123"'), /["%]/); + }); + + it('unwraps a weak etag', () => { + assert.strictEqual(formatSourceStamp(bus, 'W/"abc123"'), 'sb.abc123'); + }); + + it('keeps a multipart etag suffix', () => { + assert.strictEqual(formatSourceStamp(bus, '"abc123-7"'), 'sb.abc123-7'); + }); + + it('stamps a source-bus read that found nothing, so the write can only create', () => { + assert.strictEqual(formatSourceStamp(bus, undefined), 'sb.new'); + }); + + it('stamps a source-bus read with no etag as a bare source-bus read', () => { + assert.strictEqual(formatSourceStamp(bus, null, true), 'sb'); + }); + + it('falls back to a bare source-bus read for an etag it cannot put in a url', () => { + assert.strictEqual(formatSourceStamp(bus, '"has spaces and /"', true), 'sb'); + }); + + it('stamps a legacy read, which has no etag to carry', () => { + assert.strictEqual(formatSourceStamp(legacy, undefined), 'da'); + }); + + it('stamps a legacy read the same whether or not the document was found', () => { + assert.strictEqual(formatSourceStamp(legacy, undefined, true), 'da'); + }); +}); + +describe('parseSourceStamp', () => { + describe('a source-bus stamp', () => { + it('reads the store back', () => { + assert.strictEqual(parseSourceStamp('sb.abc123').kind, SOURCE_BUS); + }); + + it('turns the etag into an If-Match, so a changed page is refused', () => { + assert.deepStrictEqual(parseSourceStamp('sb.abc123').condition, { 'If-Match': '"abc123"' }); + }); + + it('turns a new-page stamp into If-None-Match, so an existing page is refused', () => { + assert.deepStrictEqual(parseSourceStamp('sb.new').condition, { 'If-None-Match': '*' }); + }); + + it('turns a bare stamp into If-Match: *, so it can overwrite but not create', () => { + assert.deepStrictEqual(parseSourceStamp('sb').condition, { 'If-Match': '*' }); + }); + }); + + describe('a legacy stamp', () => { + it('reads the store back', () => { + assert.strictEqual(parseSourceStamp('da').kind, LEGACY); + }); + + // da-admin sets no etag on a source GET or HEAD, only on a POST response, so a read there + // cannot produce a precondition. Verified live on 2026-08-03. + it('carries no precondition, since a legacy read yields no etag', () => { + assert.strictEqual(parseSourceStamp('da').condition, undefined); + }); + }); + + describe('anything else', () => { + [ + ['no stamp at all', null], + ['an empty stamp', ''], + ['an unknown store', 'gcs.abc'], + ['a stamp shaped like a path', 'sb/abc'], + ['an etag with a url-unsafe character', 'sb.a"b'], + ['an etag with a slash', 'sb.a/b'], + ['a stamp with an empty etag', 'sb.'], + ['a stamp trying to inject a header', 'sb.abc\r\nX-Evil: 1'], + ].forEach(([what, value]) => { + it(`is not trusted: ${what}`, () => { + assert.strictEqual(parseSourceStamp(value), undefined); + }); + }); + }); + + describe('round trip', () => { + it('parses back what a source-bus read stamped', () => { + const stamp = formatSourceStamp(bus, '"9e8311043aab12b156073d30f8bb3710"'); + + assert.deepStrictEqual(parseSourceStamp(stamp), { + kind: SOURCE_BUS, + condition: { 'If-Match': '"9e8311043aab12b156073d30f8bb3710"' }, + }); + }); + + it('parses back what a legacy read stamped', () => { + assert.deepStrictEqual(parseSourceStamp(formatSourceStamp(legacy)), { + kind: LEGACY, + condition: undefined, + }); + }); + }); +}); From a80ee1d4b227d70785c287f492c690f3e75bbc8c Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:09:25 +0200 Subject: [PATCH 16/48] feat: add the source stamp carries the store and, on the source bus, the etag the read returned, so the write can be made conditional on it. --- src/utils/source-stamp.js | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 src/utils/source-stamp.js diff --git a/src/utils/source-stamp.js b/src/utils/source-stamp.js new file mode 100644 index 00000000..21eba3ea --- /dev/null +++ b/src/utils/source-stamp.js @@ -0,0 +1,74 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { LEGACY, SOURCE_BUS } from '../storage/content-source.js'; + +/** + * The query param that carries the stamp on the Universal Editor connection uri. + * + * A read and its save are two requests, and a fresh probe on the save can disagree with the one + * that served the read. The Universal Editor Service fetches and posts back to + * `editable.connection.uri.toString()` verbatim, so a param on that uri is what links them. + */ +export const SOURCE_STAMP_PARAM = 'ab-src'; + +const SOURCE_BUS_STAMP = 'sb'; +const LEGACY_STAMP = 'da'; +const NEW_DOCUMENT = 'new'; +// what may go in a url and come back meaning the same thing +const ETAG = /^[A-Za-z0-9._~-]+$/; + +function bareEtag(etag) { + return etag?.replace(/^W\//, '').replace(/"/g, ''); +} + +/** + * Stamps a read with the store it came from and, where the store gave one, the version it read. + * + * Only the source bus sets an etag on a read, and it is the store holding the content a wrong + * write would destroy, so that is where the version matters. + * + * @param {{kind: string}} source the resolved content source + * @param {string} [etag] the etag the read returned + * @param {boolean} [found] whether the read found a document + * @returns {string} the stamp + */ +export function formatSourceStamp(source, etag, found = false) { + if (source.kind !== SOURCE_BUS) return LEGACY_STAMP; + const bare = bareEtag(etag); + if (bare && ETAG.test(bare)) return `${SOURCE_BUS_STAMP}.${bare}`; + // no usable etag: either the document is not there yet, so the write may only create it, or it + // is there but unversioned, so the write may only overwrite it + return found ? SOURCE_BUS_STAMP : `${SOURCE_BUS_STAMP}.${NEW_DOCUMENT}`; +} + +/** + * Reads a stamp back into the store it names and the precondition a write to it carries. + * + * The stamp arrives on a client-supplied url, so nothing is taken on trust: an etag that does not + * look like an etag makes the whole stamp unusable rather than being passed to a store. + * + * @param {string} [value] the raw param value + * @returns {{kind: string, condition?: Object}|undefined} undefined when there is no stamp to + * trust, which leaves the fresh probe to decide on its own + */ +export function parseSourceStamp(value) { + if (!value) return undefined; + if (value === LEGACY_STAMP) return { kind: LEGACY, condition: undefined }; + if (value === SOURCE_BUS_STAMP) return { kind: SOURCE_BUS, condition: { 'If-Match': '*' } }; + + const [store, ...rest] = value.split('.'); + const etag = rest.join('.'); + if (store !== SOURCE_BUS_STAMP || !etag) return undefined; + if (etag === NEW_DOCUMENT) return { kind: SOURCE_BUS, condition: { 'If-None-Match': '*' } }; + if (!ETAG.test(etag)) return undefined; + return { kind: SOURCE_BUS, condition: { 'If-Match': `"${etag}"` } }; +} From 3cfecf1c53baeadf053d669ac15158d830323e8c Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:11:04 +0200 Subject: [PATCH 17/48] test: pin routed reads and the stamp they leave 503 when the source is unresolved, only 404 means absent, and a UE read stamps the connection uri with the store and etag it read. --- test/routes/source-read.test.js | 340 ++++++++++++++++++++++++++ test/ue/source-stamp-scaffold.test.js | 75 ++++++ 2 files changed, 415 insertions(+) create mode 100644 test/routes/source-read.test.js create mode 100644 test/ue/source-stamp-scaffold.test.js diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js new file mode 100644 index 00000000..e29d8a2f --- /dev/null +++ b/test/routes/source-read.test.js @@ -0,0 +1,340 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import esmock from 'esmock'; +import { getDaCtx } from '../../src/utils/daCtx.js'; + +const LEGACY_SOURCE = { kind: 'legacy' }; +const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; +const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the config service answered 503' }; + +const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer t' } }); + +/** + * Builds the route module with the network replaced. `bus` answers the source bus, `legacy` + * answers da-admin, and every request to each is recorded so a test can assert where a read + * went and what it carried. + */ +const build = async ({ + source = LEGACY_SOURCE, + bus = () => new Response('from the source bus', { status: 200, headers: { etag: '"busetag"' } }), + legacy = () => new Response('from da-admin', { status: 200 }), + headHtml = '', +} = {}) => { + const seen = { bus: [], legacy: [], stamps: [] }; + globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + seen.bus.push({ url: request.url, method: request.method, headers: request.headers }); + return bus(request); + }; + const env = { + DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', + daadmin: { + fetch: async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + seen.legacy.push({ url: request.url, method: request.method, headers: request.headers }); + return legacy(request); + }, + }, + }; + const mod = await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => source, + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, + '../../src/utils/aemCtx.js': { + getAemCtx: () => ({}), + getAEMHtml: async () => headHtml, + }, + '../../src/render/compose.js': { + composeHtml: async (daCtx, aemCtx, bodyHtml) => ({ bodyHtml }), + serializeHtml: (tree) => `${tree.bodyHtml}`, + }, + '../../src/ue/ue.js': { + applyUEInstrumentation: async (tree, daCtx, aemCtx, stamp) => { seen.stamps.push(stamp); }, + }, + '../../src/storage/config.js': { + getSiteConfig: async () => { throw new Error('no config'); }, + }, + }); + return { ...mod, env, seen }; +}; + +afterEach(() => { + delete globalThis.fetch; +}); + +describe('reading with the content source resolved', () => { + describe('an html read on a source-bus site', () => { + it('reads from the base the config named', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.legacy.length, 0); + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); + }); + + it('composes the source-bus document, not da-admin\'s copy of it', async () => { + const { daSourceGet, env } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(await res.text(), 'from the source bus'); + }); + + it('forwards the author token to the source bus', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.bus[0].headers.get('Authorization'), 'Bearer t'); + }); + }); + + describe('an html read on a legacy site', () => { + it('reads from da-admin over the service binding', async () => { + const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy.length, 1); + assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); + }); + }); + + describe('when the content source could not be resolved', () => { + // guessing reads past a migrated site's live page and hands the author its stale + // pre-migration copy, which the next save would then be based on + it('refuses an html read with 503 rather than guessing a store', async () => { + const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + }); + + it('asks the caller to retry', async () => { + const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.ok(Number(res.headers.get('Retry-After')) > 0); + }); + + it('touches neither store', async () => { + const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy.length, 0); + }); + + it('refuses a non-html read too', async () => { + const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + it('refuses a HEAD with 503 and no body', async () => { + const { daSourceHead, env } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(await res.text(), ''); + }); + }); + + describe('what a store status means', () => { + // turning any of these into the blank starter template at HTTP 200 hands the author an + // empty document to save over a page that exists + [401, 403, 429, 500, 502].forEach((status) => { + it(`passes a ${status} from the store through as itself`, async () => { + const { daSourceGet, env } = await build({ + source: BUS_SOURCE, + bus: () => new Response('upstream said no', { status }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, status); + }); + }); + + it('composes the starter template on a 404, which is the one absent answer', async () => { + const { daSourceGet, env } = await build({ + source: BUS_SOURCE, + bus: () => new Response('', { status: 404 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 200); + assert.ok(!(await res.text()).includes('from the source bus')); + }); + }); + + describe('the stamp a read leaves for its write', () => { + it('carries the source-bus etag it read', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.deepStrictEqual(seen.stamps, ['sb.busetag']); + }); + + it('says the source-bus document is new when the read found nothing', async () => { + const { daSourceGet, env, seen } = await build({ + source: BUS_SOURCE, + bus: () => new Response('', { status: 404 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.deepStrictEqual(seen.stamps, ['sb.new']); + }); + + it('says only the store when a source-bus read carried no etag', async () => { + const { daSourceGet, env, seen } = await build({ + source: BUS_SOURCE, + bus: () => new Response('x', { status: 200 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.deepStrictEqual(seen.stamps, ['sb']); + }); + + it('names da-admin for a legacy read', async () => { + const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.deepStrictEqual(seen.stamps, ['da']); + }); + + it('is not applied outside UE, where nothing posts back', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.preview.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.deepStrictEqual(seen.stamps, []); + }); + }); + + describe('a non-html read', () => { + it('goes to the source bus with the case it stored the file under', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Media/holiday.PNG'); + }); + + it('goes to da-admin fully lowercased on a legacy site', async () => { + const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/media/holiday.png'); + }); + }); + + describe('a HEAD', () => { + it('goes to the source bus on a source-bus site', async () => { + const { daSourceHead, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].method, 'HEAD'); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); + }); + + it('goes to da-admin on a legacy site', async () => { + const { daSourceHead, env, seen } = await build({ source: LEGACY_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.legacy.length, 1); + assert.strictEqual(seen.legacy[0].method, 'HEAD'); + }); + + it('passes a 404 from the store through, since HEAD composes nothing', async () => { + const { daSourceHead, env } = await build({ + source: BUS_SOURCE, + bus: () => new Response('', { status: 404 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 404); + }); + }); + + describe('a read that cannot be placed at all', () => { + it('never asks a store when the hostname named no site', async () => { + const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://xyz.ue.da.live/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + }); + + describe('the order of the two things that can fail', () => { + // a missing AEM branch is answered as it was before, so quick-edit still gets its shell + it('reports a missing AEM branch even when the source is unresolved', async () => { + const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE, headHtml: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 404); + }); + }); +}); diff --git a/test/ue/source-stamp-scaffold.test.js b/test/ue/source-stamp-scaffold.test.js new file mode 100644 index 00000000..48474786 --- /dev/null +++ b/test/ue/source-stamp-scaffold.test.js @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import { getUEHtmlHeadEntries } from '../../src/ue/scaffold.js'; +import { getAemCtx } from '../../src/utils/aemCtx.js'; +import { UNAUTHORIZED_HTML_MESSAGE } from '../../src/utils/constants.js'; + +const env = { UE_HOST: 'test-ue-host', UE_SERVICE: 'test-ue-service' }; + +const connectionContent = (daCtx, stamp) => { + const entries = getUEHtmlHeadEntries(daCtx, getAemCtx(env, daCtx), stamp); + return entries.find((e) => e.properties?.name === 'urn:adobe:aue:system:ab').properties.content; +}; + +const hosted = { + org: 'org', site: 'site', ref: 'ref', path: '/some-path', aemPathname: '/some-path', +}; +const local = { ...hosted, isLocal: true, orgSiteInPath: true }; + +describe('the stamp on the UE connection uri', () => { + it('is absent when the read left none', () => { + assert.strictEqual( + connectionContent(hosted, undefined), + 'da:https://ref--site--org.test-ue-host/some-path', + ); + }); + + it('is carried as a query param the Universal Editor Service posts back', () => { + assert.strictEqual( + connectionContent(hosted, 'sb.abc123'), + 'da:https://ref--site--org.test-ue-host/some-path?ab-src=sb.abc123', + ); + }); + + it('is carried on the localhost form too', () => { + assert.strictEqual( + connectionContent(local, 'da'), + 'da:https://test-ue-host/org/site/some-path?ab-src=da', + ); + }); + + it('keeps the uri parseable, so new URL() round-trips it', () => { + const content = connectionContent(hosted, 'sb.abc123'); + const uri = content.replace(/^da:/, ''); + + assert.strictEqual(new URL(uri).toString(), uri); + assert.strictEqual(new URL(uri).searchParams.get('ab-src'), 'sb.abc123'); + }); + + it('leaves the path and host untouched, so gimme_cookie still resolves', () => { + const uri = new URL(connectionContent(hosted, 'sb.abc123').replace(/^da:/, '')); + + assert.strictEqual(uri.pathname, '/some-path'); + assert.strictEqual(uri.hostname, 'ref--site--org.test-ue-host'); + assert.strictEqual(new URL('/gimme_cookie', uri).toString(), 'https://ref--site--org.test-ue-host/gimme_cookie'); + }); + + it('is not added to the 401 sentinel, which the authorbus extension matches exactly', () => { + // the shipped extension compares the endpoint to the literal '401' and 'da://401' and on a + // match refetches /gimme_cookie and refreshes the page; a stamp would stop that firing + assert.ok(UNAUTHORIZED_HTML_MESSAGE.includes('content="da:401"')); + assert.ok(!UNAUTHORIZED_HTML_MESSAGE.includes('ab-src')); + }); +}); From b1a8c9077e89fa3f0131437c0be4dddcd09243f3 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:14:22 +0200 Subject: [PATCH 18/48] test: pin that an unusable admin host answers unknown a throw escapes into withCorsHeaders, which reads response.headers and throws again. --- test/storage/content-source.test.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index cb052ad8..edcde331 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -205,6 +205,24 @@ describe('resolveContentSource', () => { }); describe('the admin host', () => { + // the caller turns unknown into a 503 it can return; a throw here escapes into + // withCorsHeaders, which reads response.headers and throws again on undefined + it('answers unknown rather than throwing when it is not set', async () => { + stubFetch(legacyBody); + + const source = await resolveContentSource({}, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown rather than throwing when it is not a url', async () => { + stubFetch(legacyBody); + + const source = await resolveContentSource({ HLX_ADMIN: 'not-a-url' }, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + it('comes from env, so stage can point elsewhere', async () => { stubFetch(legacyBody); From a54a5b5453585073bf76bb19f704be61797b5708 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:15:27 +0200 Subject: [PATCH 19/48] feat: route reads to the store that holds the site 503 when the content source is unresolved, since guessing serves a migrated site its stale copy. only 404 means the document is absent. a UE read stamps the connection uri with the store and etag it read, which the write that follows uses as its precondition. the write path routes here too and gains that precondition in the next commit. --- src/responses/index.js | 30 +++++++++ src/routes/da-admin.js | 109 ++++++++++++++++++++------------ src/storage/content-source.js | 8 ++- src/ue/scaffold.js | 8 ++- src/ue/ue.js | 6 +- src/utils/constants.js | 6 ++ test/routes/da-admin.test.js | 40 ++++++++++++ test/routes/source-read.test.js | 15 +++-- 8 files changed, 172 insertions(+), 50 deletions(-) diff --git a/src/responses/index.js b/src/responses/index.js index 1c34e53e..8617cfa9 100644 --- a/src/responses/index.js +++ b/src/responses/index.js @@ -11,6 +11,8 @@ */ import { DEFAULT_UNAUTHORIZED_HTML_MESSAGE } from '../utils/constants.js'; +const RETRY_AFTER_SECONDS = '5'; + export function daResp({ body, status, contentType, contentLength, headers: extraHeaders, }) { @@ -47,10 +49,38 @@ export function get415(message = '') { return daResp({ body: message, status: 415, contentType: 'text/html' }); } +export function get503(message = '') { + return daResp({ + body: message, + status: 503, + contentType: 'text/html', + headers: [['Retry-After', RETRY_AFTER_SECONDS]], + }); +} + +// a refused write is never rendered. The Universal Editor Service embeds the body verbatim in +// its problem+json error string, so plain text is what an author is shown. +export function post409(message = '') { + return daResp({ body: message, status: 409, contentType: 'text/plain; charset=utf-8' }); +} + +export function post503(message = '') { + return daResp({ + body: message, + status: 503, + contentType: 'text/plain; charset=utf-8', + headers: [['Retry-After', RETRY_AFTER_SECONDS]], + }); +} + export function head401() { return new Response(null, { status: 401 }); } +export function head503() { + return new Response(null, { status: 503, headers: { 'Retry-After': RETRY_AFTER_SECONDS } }); +} + export function head404() { return new Response(null, { status: 404 }); } diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index 65f2508e..f20bcf30 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -22,10 +22,19 @@ import { applyQuickEditToDocument, buildQuickEditCookie, buildQuickEditNotFoundResponse, } from '../utils/quick-edit.js'; import { - daResp, get401, get404, get415, head401, + daResp, get401, get404, get415, get503, head401, head503, post503, } from '../responses/index.js'; -import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, UNAUTHORIZED_HTML_MESSAGE } from '../utils/constants.js'; +import { + BRANCH_NOT_FOUND_HTML_MESSAGE, + DEFAULT_HTML_TEMPLATE, + SOURCE_UNRESOLVED_HTML_MESSAGE, + SOURCE_UNRESOLVED_MESSAGE, + UNAUTHORIZED_HTML_MESSAGE, +} from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; +import resolveContentSource, { UNKNOWN } from '../storage/content-source.js'; +import getStore from '../storage/store.js'; +import { formatSourceStamp } from '../utils/source-stamp.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; const HTML_POST_TYPE = 'text/html'; @@ -35,10 +44,6 @@ export function isHtmlPostType(type) { return type.split(';')[0].trim().toLowerCase() === HTML_POST_TYPE; } -function getSourceUrl(env, { org, site, sourcePath }) { - return new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); -} - async function getFileBody(data) { const text = await data.text(); return { body: text, type: data.type }; @@ -101,16 +106,24 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing - const adminUrl = getSourceUrl(env, daCtx); - console.log(`-> ${adminUrl.toString()}`); - const response = await env.daadmin.fetch(adminUrl, { method: 'GET', headers }); - console.log(`<- ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); + const source = await resolveContentSource(env, daCtx); + if (source.kind === UNKNOWN) { + console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); + return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); + } + const store = getStore(env, daCtx, source); + console.log(`-> ${store.url.toString()}`); + const response = await store.fetch(store.url, { method: 'GET', headers }); + console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } - // get the AEM parts (head.html) + // the store lookup costs a round trip, so it runs alongside head.html rather than after it const aemCtx = getAemCtx(env, daCtx); - const headHtml = await getAEMHtml(aemCtx, '/head.html'); + const [headHtml, source] = await Promise.all([ + getAEMHtml(aemCtx, '/head.html'), + resolveContentSource(env, daCtx), + ]); if (!headHtml) { // quick-edit still needs a working shell (with the import map) so the editor // can load into this page, even when the AEM branch doesn't exist yet. @@ -119,22 +132,33 @@ export async function daSourceGet({ req, env, daCtx }) { } return get404(BRANCH_NOT_FOUND_HTML_MESSAGE); } + if (source.kind === UNKNOWN) { + console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); + return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); + } - // get the content from DA admin - const adminUrl = getSourceUrl(env, daCtx); + // get the content from the store that holds it + const store = getStore(env, daCtx, source); // eslint-disable-next-line no-param-reassign - req = new Request(adminUrl, { + req = new Request(store.url, { method: 'GET', headers, }); - console.log(`-> ${adminUrl.toString()}`); - const daAdminResp = await env.daadmin.fetch(req); - console.log(`<- ${adminUrl.toString()}. ${daAdminResp.status} ${daAdminResp.statusText}`, { status: daAdminResp.status, statusText: daAdminResp.statusText }); + console.log(`-> ${store.url.toString()}`); + const sourceResp = await store.fetch(req); + console.log(`<- ${store.url.toString()}. ${sourceResp.status} ${sourceResp.statusText}`, { status: sourceResp.status, statusText: sourceResp.statusText }); + + // only a 404 means "this document is not here". Composing the starter template over anything + // else hands the author a blank page to save over a document that exists. + if (sourceResp.status !== 200 && sourceResp.status !== 404) { + return sourceResp; + } + const found = sourceResp.status === 200; // use the stored content when available, otherwise fall back to a template - const bodyHtml = daAdminResp && daAdminResp.status === 200 - ? await daAdminResp.text() + const bodyHtml = found + ? await sourceResp.text() : await getPageTemplate(env, daCtx, aemCtx, headHtml); // compose the page the same way for every request type @@ -150,7 +174,9 @@ export async function daSourceGet({ req, env, daCtx }) { extraHeaders.push(['Set-Cookie', buildQuickEditCookie(entryPath)]); } } else if (isUE) { - await applyUEInstrumentation(documentTree, daCtx, aemCtx); + // UE is the only client that posts back, so it is the only one that needs the stamp + const stamp = formatSourceStamp(source, sourceResp.headers.get('etag'), found); + await applyUEInstrumentation(documentTree, daCtx, aemCtx, stamp); } const body = serializeHtml(documentTree); @@ -174,10 +200,16 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const adminUrl = getSourceUrl(env, daCtx); - console.log(`-> HEAD ${adminUrl.toString()}`); - const response = await env.daadmin.fetch(adminUrl, { method: 'HEAD', headers }); - console.log(`<- HEAD ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); + const source = await resolveContentSource(env, daCtx); + if (source.kind === UNKNOWN) { + console.warn(`503 HEAD ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); + return head503(); + } + + const store = getStore(env, daCtx, source); + console.log(`-> HEAD ${store.url.toString()}`); + const response = await store.fetch(store.url, { method: 'HEAD', headers }); + console.log(`<- HEAD ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return new Response(null, { status: response.status, headers: response.headers }); } @@ -213,22 +245,21 @@ export async function daSourcePost({ req, env, daCtx }) { minifyWhitespace(bodyNode); - // create new POST request with the body content - const body = new FormData(); const bodyContent = toHtml(bodyNode); - const data = new Blob([bodyContent], { type: 'text/html' }); - body.set('data', data); - const headers = { Authorization: authToken }; - const adminUrl = getSourceUrl(env, daCtx); + + const source = await resolveContentSource(env, daCtx); + if (source.kind === UNKNOWN) { + console.warn(`503 POST ${sourcePath}, content source unresolved: ${source.reason}`); + return post503(SOURCE_UNRESOLVED_MESSAGE); + } + + // the two stores take the document in different shapes, so the store builds its own request + const store = getStore(env, daCtx, source); // eslint-disable-next-line no-param-reassign - req = new Request(adminUrl, { - method: 'POST', - body, - headers, - }); - console.log(`-> ${adminUrl.toString()}`); - const response = await env.daadmin.fetch(req); - console.log(`<- ${adminUrl.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); + req = new Request(store.url, store.writeInit(bodyContent, authToken)); + console.log(`-> ${store.url.toString()}`); + const response = await store.fetch(req); + console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } diff --git a/src/storage/content-source.js b/src/storage/content-source.js index 4a96abd4..2aee7dd9 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -48,7 +48,13 @@ export default async function resolveContentSource(env, daCtx) { return unknown(org, site, 'no org or site in the request'); } - const url = new URL(`/sidekick/${org}/${site}/${ref}/config.json`, env.HLX_ADMIN); + let url; + try { + url = new URL(`/sidekick/${org}/${site}/${ref}/config.json`, env.HLX_ADMIN); + } catch (e) { + return unknown(org, site, `HLX_ADMIN is not a url: ${e.message}`); + } + const headers = new Headers(); if (authToken) headers.set('Authorization', authToken); diff --git a/src/ue/scaffold.js b/src/ue/scaffold.js index 2d673abe..e0618724 100644 --- a/src/ue/scaffold.js +++ b/src/ue/scaffold.js @@ -12,8 +12,9 @@ import { h } from 'hastscript'; import { withAemAuth } from '../utils/aemCtx.js'; +import { SOURCE_STAMP_PARAM } from '../utils/source-stamp.js'; -export function getUEHtmlHeadEntries(daCtx, aemCtx) { +export function getUEHtmlHeadEntries(daCtx, aemCtx, sourceStamp) { const { org, site, @@ -28,9 +29,12 @@ export function getUEHtmlHeadEntries(daCtx, aemCtx) { let finalUeService = ueService; const children = []; + // the Universal Editor Service reads this uri and posts back to it verbatim, so the stamp on + // it is what tells a save which store the content it is saving was read from + const stamp = sourceStamp ? `?${SOURCE_STAMP_PARAM}=${sourceStamp}` : ''; children.push(h('meta', { name: 'urn:adobe:aue:system:ab', - content: isLocal ? `da:https://${ueHostname}/${org}/${site}${path}` : `da:https://${ref}--${site}--${org}.${ueHostname}${path}`, + content: isLocal ? `da:https://${ueHostname}/${org}/${site}${path}${stamp}` : `da:https://${ref}--${site}--${org}.${ueHostname}${path}${stamp}`, })); if (ueServiceParam && ueServiceParam === 'local') { diff --git a/src/ue/ue.js b/src/ue/ue.js index e03b5118..d9a3961b 100644 --- a/src/ue/ue.js +++ b/src/ue/ue.js @@ -21,11 +21,13 @@ import { injectUEAttributes } from './attributes.js'; * @param {import('hast').Root} documentTree - The composed document tree (mutated in place). * @param {Object} daCtx - The Dark Alley context object. * @param {Object} aemCtx - The AEM context object. + * @param {string} [sourceStamp] - The store and version this page was read from, carried on the + * connection uri so the save that follows lands in the same place. */ -export async function applyUEInstrumentation(documentTree, daCtx, aemCtx) { +export async function applyUEInstrumentation(documentTree, daCtx, aemCtx, sourceStamp) { // add UE head script and meta tags const headNode = select('head', documentTree); - headNode.children.push(...getUEHtmlHeadEntries(daCtx, aemCtx)); + headNode.children.push(...getUEHtmlHeadEntries(daCtx, aemCtx, sourceStamp)); // add data attributes for UE to the body const bodyNode = select('body', documentTree); diff --git a/src/utils/constants.js b/src/utils/constants.js index 138c246a..36b55f96 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -51,4 +51,10 @@ export const DEFAULT_HTML_TEMPLATE = '

Not found: Unable to retrieve AEM branch

'; +export const SOURCE_UNRESOLVED_HTML_MESSAGE = '

503: Content source unresolved

The store that holds this document could not be determined. Please retry.

'; + +export const SOURCE_UNRESOLVED_MESSAGE = 'The store that holds this document could not be determined, so the write was refused rather than sent to the wrong one. Please retry.'; + +export const SOURCE_MOVED_MESSAGE = 'This document moved to a different content store while it was open. Reload the page to pick up the new one; saving now would write to the store it left.'; + export const DEFAULT_UNAUTHORIZED_HTML_MESSAGE = '

401: Unauthorized

'; diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index 777c3e31..97c10546 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -32,6 +32,7 @@ const recorder = () => { const fetched = []; const env = { DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', daadmin: { fetch: async (input) => { fetched.push(input instanceof Request ? input.url : input.href); @@ -43,6 +44,12 @@ const recorder = () => { }; const mockRoutes = async () => esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => ({ kind: 'legacy' }), + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({}), getAEMHtml: async () => '', @@ -80,6 +87,7 @@ describe('daSourceHead', () => { describe('daSourceGet', () => { const env = { DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', daadmin: { fetch: async () => new Response('stored', { status: 200 }) }, }; @@ -93,6 +101,12 @@ describe('daSourceGet', () => { const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; calls = { compose: [], ue: 0, quickEdit: 0 }; return (await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => ({ kind: 'legacy' }), + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({}), getAEMHtml: async () => headHtml, @@ -241,6 +255,19 @@ describe('daSourceGet', () => { }); describe('source URLs', () => { + // these drive the unmocked module, so the content-source lookup really does reach out; + // answer it as the legacy store, which is the store these tests describe + beforeEach(() => { + globalThis.fetch = async () => new Response( + JSON.stringify({ contentSourceUrl: 'https://content.da.live/org/site/' }), + { status: 200 }, + ); + }); + + afterEach(() => { + delete globalThis.fetch; + }); + it('GET / reads /index.html', async () => { const { daSourceGet } = await mockRoutes(); const { env, fetched } = recorder(); @@ -369,6 +396,19 @@ describe('daSourcePost to a non-HTML path', () => { }); describe('daSourcePost', () => { + // these drive the unmocked module, so the content-source lookup really does reach out; + // answer it as the legacy store, which is the store these tests describe + beforeEach(() => { + globalThis.fetch = async () => new Response( + JSON.stringify({ contentSourceUrl: 'https://content.da.live/org/site/' }), + { status: 200 }, + ); + }); + + afterEach(() => { + delete globalThis.fetch; + }); + // on an HTML path, so the path check does not answer first and this exercises // the part-type check it('refuses a binary File with 415 and does not write', async () => { diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index e29d8a2f..a0ecbb63 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -26,12 +26,15 @@ const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer * answers da-admin, and every request to each is recorded so a test can assert where a read * went and what it carried. */ -const build = async ({ - source = LEGACY_SOURCE, - bus = () => new Response('from the source bus', { status: 200, headers: { etag: '"busetag"' } }), - legacy = () => new Response('from da-admin', { status: 200 }), - headHtml = '', -} = {}) => { +const build = async (overrides = {}) => { + const { + source = LEGACY_SOURCE, + bus = () => new Response('from the source bus', { status: 200, headers: { etag: '"busetag"' } }), + legacy = () => new Response('from da-admin', { status: 200 }), + } = overrides; + // 'headHtml' in overrides rather than a destructured default, so passing + // `{ headHtml: undefined }` really does simulate a missing head.html + const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; const seen = { bus: [], legacy: [], stamps: [] }; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); From 0f90abb0e77b8284499a61e8d2b0ae7aee498d4d Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:16:26 +0200 Subject: [PATCH 20/48] test: pin conditional writes and the store-moved refusal a source-bus save carries the etag its read returned, so a wrong-store or stale write is refused by the store rather than landing. 409 when the read and the save find different stores. --- test/routes/source-write.test.js | 290 +++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 test/routes/source-write.test.js diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js new file mode 100644 index 00000000..66aeab6b --- /dev/null +++ b/test/routes/source-write.test.js @@ -0,0 +1,290 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import esmock from 'esmock'; +import { getDaCtx } from '../../src/utils/daCtx.js'; + +const LEGACY_SOURCE = { kind: 'legacy' }; +const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; +const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the config service answered 503' }; + +const DOC = '

the author typed this

'; + +/** The shape the Universal Editor Service posts: a `data` blob in a multipart form. */ +const uePost = (url, html = DOC) => { + const body = new FormData(); + body.set('data', new File([html], 'content.html', { type: 'text/html' })); + return new Request(url, { method: 'POST', body, headers: { Authorization: 'Bearer t' } }); +}; + +const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { + const seen = { bus: [], legacy: [] }; + const capture = async (request) => { + const clone = request.clone(); + return { + url: request.url, + method: request.method, + headers: request.headers, + body: await clone.text(), + contentType: request.headers.get('Content-Type'), + }; + }; + globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + seen.bus.push(await capture(request)); + return new Response('', { status }); + }; + const env = { + DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', + daadmin: { + fetch: async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + seen.legacy.push(await capture(request)); + return new Response('', { status }); + }, + }, + }; + const mod = await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => source, + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, + }); + return { daSourcePost: mod.daSourcePost, env, seen }; +}; + +const post = async (opts, url) => { + const { daSourcePost, env, seen } = await build(opts); + const req = uePost(url); + const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + return { res, seen }; +}; + +const AT = 'https://main--site--org.ue.da.live/folder/content'; + +afterEach(() => { + delete globalThis.fetch; +}); + +describe('writing with the content source resolved', () => { + describe('a stamped source-bus save', () => { + it('goes to the source bus', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + + assert.strictEqual(seen.legacy.length, 0); + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); + }); + + // this is the whole point: the etag came from the read, so a write conditioned on it cannot + // land on a document the author never saw + it('carries the etag the read returned as an If-Match', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + + assert.strictEqual(seen.bus[0].headers.get('If-Match'), '"busetag"'); + }); + + it('sends the document as the raw body the source bus parses', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + + assert.strictEqual(seen.bus[0].body, DOC); + assert.strictEqual(seen.bus[0].contentType, 'text/html'); + }); + + // helix-api-service parses no form data; the envelope would be stored as the document text + it('never wraps the body in a multipart envelope', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + + assert.ok(!seen.bus[0].body.includes('Content-Disposition'), seen.bus[0].body); + assert.ok(!/boundary/i.test(seen.bus[0].contentType ?? ''), seen.bus[0].contentType); + }); + + it('passes the store answer back, so a 412 reaches the editor', async () => { + const { res } = await post({ source: BUS_SOURCE, status: 412 }, `${AT}?ab-src=sb.busetag`); + + assert.strictEqual(res.status, 412); + }); + }); + + describe('a save of a page the read did not find', () => { + it('may only create, so an existing page is refused by the store', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.new`); + + assert.strictEqual(seen.bus[0].headers.get('If-None-Match'), '*'); + assert.strictEqual(seen.bus[0].headers.get('If-Match'), null); + }); + }); + + describe('a source-bus save whose read carried no etag', () => { + it('may overwrite but not create', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); + + assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); + }); + }); + + describe('a stamped legacy save', () => { + it('goes to da-admin as a data form part', async () => { + const { seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=da`); + + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); + assert.ok(seen.legacy[0].body.includes('name="data"'), seen.legacy[0].body); + assert.ok(seen.legacy[0].body.includes('the author typed this'), seen.legacy[0].body); + }); + + // da-admin sets no etag on a read, so there is nothing to condition on + it('carries no precondition', async () => { + const { seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=da`); + + assert.strictEqual(seen.legacy[0].headers.get('If-Match'), null); + assert.strictEqual(seen.legacy[0].headers.get('If-None-Match'), null); + }); + }); + + describe('when the store moved between the read and the save', () => { + // the destructive case. The author read da-admin's copy, which for a migrated site is its + // stale pre-migration content, and the site is now served from the source bus. Writing it + // there would overwrite the live page with content the author never saw. + it('refuses a legacy-stamped save to a site now on the source bus', async () => { + const { res, seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=da`); + + assert.strictEqual(res.status, 409); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + it('refuses a source-bus-stamped save to a site now on da-admin', async () => { + const { res, seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=sb.busetag`); + + assert.strictEqual(res.status, 409); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + it('says so in plain text, since nothing renders a refused write', async () => { + const { res } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=da`); + + assert.match(res.headers.get('Content-Type'), /^text\/plain/); + assert.ok((await res.text()).length > 0); + }); + }); + + describe('a save with no stamp', () => { + it('goes to da-admin unconditionally on a legacy site, as it did before', async () => { + const { res, seen } = await post({ source: LEGACY_SOURCE }, AT); + + assert.strictEqual(res.status, 201); + assert.strictEqual(seen.legacy.length, 1); + assert.strictEqual(seen.legacy[0].headers.get('If-Match'), null); + }); + + // an old page, or a stamp the editor dropped. There is no provenance, so the save may + // overwrite a page that exists but may not invent a new one. + it('may overwrite but not create on a source-bus site', async () => { + const { seen } = await post({ source: BUS_SOURCE }, AT); + + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); + }); + }); + + describe('a save carrying a stamp that cannot be trusted', () => { + [ + ['an unknown store', 'gcs.abc'], + ['an etag with a quote in it', 'sb.a"b'], + ['a header injection attempt', 'sb.abc%0d%0aX-Evil:%201'], + ['an empty value', ''], + ].forEach(([what, value]) => { + it(`falls back to no stamp: ${what}`, async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=${value}`); + + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); + }); + }); + + it('never lets a stamp put a raw newline in a header', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.a%0db`); + + assert.ok(!/[\r\n]/.test(seen.bus[0].headers.get('If-Match') ?? '')); + }); + }); + + describe('when the content source could not be resolved', () => { + it('refuses with 503 and touches neither store', async () => { + const { res, seen } = await post({ source: UNKNOWN_SOURCE }, `${AT}?ab-src=sb.busetag`); + + assert.strictEqual(res.status, 503); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + it('asks the caller to retry', async () => { + const { res } = await post({ source: UNKNOWN_SOURCE }, AT); + + assert.ok(Number(res.headers.get('Retry-After')) > 0); + }); + + it('says so in plain text', async () => { + const { res } = await post({ source: UNKNOWN_SOURCE }, AT); + + assert.match(res.headers.get('Content-Type'), /^text\/plain/); + }); + }); + + describe('what is written', () => { + it('strips the UE data attributes before the store sees them', async () => { + const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); + const req = uePost( + `${AT}?ab-src=sb.busetag`, + '

text

', + ); + + await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + + assert.ok(!seen.bus[0].body.includes('data-aue-resource'), seen.bus[0].body); + }); + + it('refuses a non-html path before resolving anything', async () => { + const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); + const req = uePost('https://main--site--org.ue.da.live/folder/data.json'); + + const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 415); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + }); + + describe('the path a save is written to', () => { + it('keeps the case the source bus stores it under', async () => { + const { seen } = await post( + { source: BUS_SOURCE }, + 'https://main--site--org.ue.da.live/Folder/Content?ab-src=sb.busetag', + ); + + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Folder/content.html'); + }); + + it('lowercases the whole path for da-admin', async () => { + const { seen } = await post( + { source: LEGACY_SOURCE }, + 'https://main--site--org.ue.da.live/Folder/Content?ab-src=da', + ); + + assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); + }); + }); +}); From eabea903a1a44b57be7d57981fe105a26f90fed5 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:17:15 +0200 Subject: [PATCH 21/48] feat: make a source-bus write conditional on the read that fed it the stamp the read left carries the etag, so a save conditioned on it cannot land on a document the author never saw: a wrong store or a changed page is refused by the store with 412. where the stamp and a fresh lookup name different stores the site moved while the page was open, which is answered 409. --- src/routes/da-admin.js | 29 ++++++++++++++++++++++++----- src/utils/daCtx.js | 6 ++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index f20bcf30..8ac60239 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -22,19 +22,20 @@ import { applyQuickEditToDocument, buildQuickEditCookie, buildQuickEditNotFoundResponse, } from '../utils/quick-edit.js'; import { - daResp, get401, get404, get415, get503, head401, head503, post503, + daResp, get401, get404, get415, get503, head401, head503, post409, post503, } from '../responses/index.js'; import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, + SOURCE_MOVED_MESSAGE, SOURCE_UNRESOLVED_HTML_MESSAGE, SOURCE_UNRESOLVED_MESSAGE, UNAUTHORIZED_HTML_MESSAGE, } from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; -import resolveContentSource, { UNKNOWN } from '../storage/content-source.js'; +import resolveContentSource, { SOURCE_BUS, UNKNOWN } from '../storage/content-source.js'; import getStore from '../storage/store.js'; -import { formatSourceStamp } from '../utils/source-stamp.js'; +import { formatSourceStamp, parseSourceStamp } from '../utils/source-stamp.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; const HTML_POST_TYPE = 'text/html'; @@ -247,17 +248,35 @@ export async function daSourcePost({ req, env, daCtx }) { const bodyContent = toHtml(bodyNode); + // the payload is settled, so the only question left is where it goes. A write is the one + // operation a wrong guess cannot be walked back from. const source = await resolveContentSource(env, daCtx); if (source.kind === UNKNOWN) { console.warn(`503 POST ${sourcePath}, content source unresolved: ${source.reason}`); return post503(SOURCE_UNRESOLVED_MESSAGE); } + // the read stamped the connection uri with the store it came from, and the Universal Editor + // Service posts back to that uri, so the two requests are linked. Where the stamp and a + // fresh lookup disagree, the site moved stores while the page was open: writing to the store + // the content came from orphans it, writing to the new one overwrites a page the author + // never saw, and neither is worth doing silently. + const stamp = parseSourceStamp(daCtx.sourceStamp); + if (stamp && stamp.kind !== source.kind) { + console.warn(`409 POST ${sourcePath}, read from ${stamp.kind} but the site is on ${source.kind}`); + return post409(SOURCE_MOVED_MESSAGE); + } + + // with no stamp there is no provenance, so a source-bus write may overwrite a page that + // exists but may not invent one + const condition = stamp?.condition + ?? (source.kind === SOURCE_BUS ? { 'If-Match': '*' } : undefined); + // the two stores take the document in different shapes, so the store builds its own request const store = getStore(env, daCtx, source); // eslint-disable-next-line no-param-reassign - req = new Request(store.url, store.writeInit(bodyContent, authToken)); - console.log(`-> ${store.url.toString()}`); + req = new Request(store.url, store.writeInit(bodyContent, authToken, condition)); + console.log(`-> ${store.url.toString()}${condition ? ` ${JSON.stringify(condition)}` : ''}`); const response = await store.fetch(req); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; diff --git a/src/utils/daCtx.js b/src/utils/daCtx.js index 42076717..12e4d128 100644 --- a/src/utils/daCtx.js +++ b/src/utils/daCtx.js @@ -9,6 +9,7 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ +import { SOURCE_STAMP_PARAM } from './source-stamp.js'; function getRefSiteOrgPath(hostname, pathname) { if (hostname === 'localhost') { @@ -113,6 +114,11 @@ export function getDaCtx(req) { const ueService = searchParams.get('ue-service'); if (ueService !== null) daCtx.ueService = ueService; + // the store a UE read came from, stamped onto the connection uri that the Universal Editor + // Service posts back to. Raw here; only source-stamp.js decides what it may be trusted to say. + const sourceStamp = searchParams.get(SOURCE_STAMP_PARAM); + if (sourceStamp !== null) daCtx.sourceStamp = sourceStamp; + daCtx.authToken = getAuthToken(req); daCtx.siteToken = getSiteToken(req); return daCtx; From 6ff6120428a6794aac3b4401f5b99e65291848af Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:20:58 +0200 Subject: [PATCH 22/48] test: pin that a store url must open the source, not appear inside it the base is used verbatim as a store url and the author token goes with it. found by a mutation that swapped startsWith for includes and survived. --- test/storage/content-source.test.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index edcde331..e3df62f2 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -136,6 +136,24 @@ describe('resolveContentSource', () => { assert.strictEqual(source.kind, 'unknown'); }); + + // the base is used verbatim as a store url and the author token goes with it, so the store + // has to be named at the front of the url and not merely somewhere inside it + it('answers unknown when a store url appears anywhere but the start', async () => { + stubFetch(() => sidekick('https://elsewhere.example/?to=https://api.aem.live/org/sites/site/source')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + + it('answers unknown for a legacy url buried mid-string too', async () => { + stubFetch(() => sidekick('https://elsewhere.example/#https://content.da.live/org/site/')); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); }); describe('when the question could not be answered', () => { From da9016febbefbfc73961776359746921eb4d83dd Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 05:24:20 +0200 Subject: [PATCH 23/48] test: pin what a media read does when the source is unresolved an image falls through to the published copy on *.aem.page, since the handlers already race the two and an image cannot become a write. mp4 is not raced, so its refusal reaches the caller. --- test/routes/source-read.test.js | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index a0ecbb63..b564e9c3 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -329,6 +329,54 @@ describe('reading with the content source resolved', () => { }); }); + describe('a media read, which the handlers race against the AEM proxy', () => { + // getHandler races an image read against *.aem.page and takes the proxy answer whenever the + // store read is not a 200. So an unresolved source degrades an image to the published copy + // rather than breaking the page, and an image cannot be laundered into a write: a POST to a + // non-html path is refused with 415 before anything is resolved. + it('falls through to the AEM proxy for an image when the source is unresolved', async () => { + const seen = []; + globalThis.fetch = async (input) => { + seen.push(input.toString()); + return new Response('the published bytes', { status: 200 }); + }; + const getHandler = (await esmock('../../src/handlers/get.js', { + '../../src/routes/da-admin.js': { + daSourceGet: async () => new Response('', { status: 503 }), + }, + '../../src/routes/aem-proxy.js': { + handleAEMProxyRequest: async () => new Response('the published bytes', { status: 200 }), + }, + })).default; + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const res = await getHandler({ req, env: {}, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 200); + assert.strictEqual(await res.text(), 'the published bytes'); + }); + + // mp4 is not in the raced extensions, so it has no published copy to fall back to and the + // refusal reaches the caller as itself + it('refuses a video read when the source is unresolved', async () => { + const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/clip.mp4'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + }); + + it('reads a video from the source bus when the source is known', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/clip.mp4'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/clip.mp4'); + }); + }); + describe('the order of the two things that can fail', () => { // a missing AEM branch is answered as it was before, so quick-edit still gets its shell it('reports a missing AEM branch even when the source is unresolved', async () => { From 02d19e4ed8adad0d477ca3f81238537c94bd1142 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 06:02:47 +0200 Subject: [PATCH 24/48] test: pin that a UE session can save more than once the editor keeps the stamp it was served at page load and posts back to it for every edit, so a precondition pinned to a version lands the first save and refuses the rest. --- test/routes/source-write.test.js | 99 ++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 66aeab6b..866556b2 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -80,6 +80,105 @@ afterEach(() => { delete globalThis.fetch; }); +describe('a UE session, which saves many times against one page load', () => { + // The Universal Editor Service runs load() then store() for each mutating operation, against + // the connection uri the editor read once at page load + // (universal-editor-service-plugin-da/src/index.ts: add 104/135, copy 150/184, move 242/270, + // patch 301/344, remove 474/492, update 540/555). Every one returns updates[] for in-place DOM + // patching, so the iframe never reloads and the stamp is never refreshed. A precondition that + // pins a version therefore lands the first save and is refused 412 for the rest of the session. + const store = (present) => { + let etag = present ? '"v1"' : undefined; + let body = present ? 'the original' : undefined; + return { + answer: (request) => { + const ifMatch = request.headers.get('If-Match'); + const ifNone = request.headers.get('If-None-Match'); + if (request.method !== 'POST') { + return etag === undefined + ? new Response('', { status: 404 }) + : new Response(body, { status: 200, headers: { etag } }); + } + if (ifNone === '*' && etag !== undefined) return new Response('', { status: 412 }); + if (ifMatch === '*' && etag === undefined) return new Response('', { status: 412 }); + if (ifMatch && ifMatch !== '*' && ifMatch !== etag) return new Response('', { status: 412 }); + body = 'written'; + etag = `"v${Number(etag?.replace(/\D/g, '') ?? 0) + 1}"`; + return new Response('', { status: 201 }); + }, + get body() { return body; }, + }; + }; + + const session = async (present) => { + const st = store(present); + globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + return st.answer(request); + }; + const env = { + DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', + daadmin: { fetch: async () => assert.fail('a source-bus site must not touch da-admin') }, + }; + let served; + const mod = await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => BUS_SOURCE, + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, + '../../src/utils/aemCtx.js': { + getAemCtx: () => ({}), + getAEMHtml: async () => '', + }, + '../../src/render/compose.js': { + composeHtml: async () => ({}), + serializeHtml: () => 'composed', + }, + '../../src/ue/ue.js': { + applyUEInstrumentation: async (tree, daCtx, aemCtx, stamp) => { served = stamp; }, + }, + '../../src/storage/config.js': { + getSiteConfig: async () => { throw new Error('no config'); }, + }, + }); + + // the editor loads the page once and keeps whatever stamp it was served + const read = new Request(AT, { headers: { Authorization: 'Bearer t' } }); + await mod.daSourceGet({ req: read, env, daCtx: getDaCtx(read) }); + + const at = `${AT}?ab-src=${served}`; + const statuses = []; + for (let i = 0; i < 3; i += 1) { + const request = uePost(at, `

edit ${i + 1}

`); + // eslint-disable-next-line no-await-in-loop + const res = await mod.daSourcePost({ req: request, env, daCtx: getDaCtx(request) }); + statuses.push(res.status); + } + return { statuses, stored: st.body, stamp: served }; + }; + + it('lands all three saves on a page that already existed', async () => { + const { statuses, stamp } = await session(true); + + assert.deepStrictEqual(statuses, [201, 201, 201], `with the stamp the read served: ${stamp}`); + }); + + it('lands all three saves on a page it created', async () => { + const { statuses, stamp } = await session(false); + + assert.deepStrictEqual(statuses, [201, 201, 201], `with the stamp the read served: ${stamp}`); + }); + + it('keeps the last edit, not the first', async () => { + const { stored } = await session(true); + + assert.strictEqual(stored, 'written'); + }); +}); + describe('writing with the content source resolved', () => { describe('a stamped source-bus save', () => { it('goes to the source bus', async () => { From 2fa26f70e8c3a14daa849399fee969d8dd096bb1 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 06:05:44 +0200 Subject: [PATCH 25/48] fix: drop the version pin, which refused every save after the first the editor keeps the connection uri it was served at page load and posts back to it for each edit, so an If-Match on the read's etag lands the first save and 412s the rest. the source bus sets no etag on a write response either, so nothing can refresh it. the stamp now says only which store the read used and whether it found a document. store identity is still carried, by the 409 when the stamp and a fresh lookup disagree. --- src/routes/da-admin.js | 10 ++-- src/utils/source-stamp.js | 45 ++++++++---------- test/routes/source-read.test.js | 68 +++++++++++++++++++++++++-- test/routes/source-write.test.js | 48 +++++++++---------- test/utils/source-stamp.test.js | 79 +++++++++++++------------------- 5 files changed, 143 insertions(+), 107 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index 8ac60239..9847bb88 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -176,7 +176,7 @@ export async function daSourceGet({ req, env, daCtx }) { } } else if (isUE) { // UE is the only client that posts back, so it is the only one that needs the stamp - const stamp = formatSourceStamp(source, sourceResp.headers.get('etag'), found); + const stamp = formatSourceStamp(source, found); await applyUEInstrumentation(documentTree, daCtx, aemCtx, stamp); } @@ -268,9 +268,11 @@ export async function daSourcePost({ req, env, daCtx }) { } // with no stamp there is no provenance, so a source-bus write may overwrite a page that - // exists but may not invent one - const condition = stamp?.condition - ?? (source.kind === SOURCE_BUS ? { 'If-Match': '*' } : undefined); + // exists but may not invent one. A stamp that carries no precondition is not the same thing: + // it says the read looked at the source bus and found nothing, so creating is what it asked + // for. + const noProvenance = source.kind === SOURCE_BUS ? { 'If-Match': '*' } : undefined; + const condition = stamp ? stamp.condition : noProvenance; // the two stores take the document in different shapes, so the store builds its own request const store = getStore(env, daCtx, source); diff --git a/src/utils/source-stamp.js b/src/utils/source-stamp.js index 21eba3ea..2bc70c6f 100644 --- a/src/utils/source-stamp.js +++ b/src/utils/source-stamp.js @@ -14,7 +14,7 @@ import { LEGACY, SOURCE_BUS } from '../storage/content-source.js'; /** * The query param that carries the stamp on the Universal Editor connection uri. * - * A read and its save are two requests, and a fresh probe on the save can disagree with the one + * A read and its save are two requests, and a fresh lookup on the save can disagree with the one * that served the read. The Universal Editor Service fetches and posts back to * `editable.connection.uri.toString()` verbatim, so a param on that uri is what links them. */ @@ -23,52 +23,43 @@ export const SOURCE_STAMP_PARAM = 'ab-src'; const SOURCE_BUS_STAMP = 'sb'; const LEGACY_STAMP = 'da'; const NEW_DOCUMENT = 'new'; -// what may go in a url and come back meaning the same thing -const ETAG = /^[A-Za-z0-9._~-]+$/; - -function bareEtag(etag) { - return etag?.replace(/^W\//, '').replace(/"/g, ''); -} /** - * Stamps a read with the store it came from and, where the store gave one, the version it read. + * Stamps a read with the store it came from, and whether it found a document there. * - * Only the source bus sets an etag on a read, and it is the store holding the content a wrong - * write would destroy, so that is where the version matters. + * The stamp says nothing about which version was read. One page load produces many saves: the + * editor keeps the connection uri it was served and posts back to it for every edit, so a + * precondition pinned to a version would land the first save and refuse the rest. Nothing on the + * write path can refresh it either, because the source bus sets no etag on a write response. * * @param {{kind: string}} source the resolved content source - * @param {string} [etag] the etag the read returned * @param {boolean} [found] whether the read found a document * @returns {string} the stamp */ -export function formatSourceStamp(source, etag, found = false) { +export function formatSourceStamp(source, found = false) { if (source.kind !== SOURCE_BUS) return LEGACY_STAMP; - const bare = bareEtag(etag); - if (bare && ETAG.test(bare)) return `${SOURCE_BUS_STAMP}.${bare}`; - // no usable etag: either the document is not there yet, so the write may only create it, or it - // is there but unversioned, so the write may only overwrite it return found ? SOURCE_BUS_STAMP : `${SOURCE_BUS_STAMP}.${NEW_DOCUMENT}`; } /** * Reads a stamp back into the store it names and the precondition a write to it carries. * - * The stamp arrives on a client-supplied url, so nothing is taken on trust: an etag that does not - * look like an etag makes the whole stamp unusable rather than being passed to a store. + * Each precondition holds for every save in a session, not only the first. `If-Match: *` asks the + * source bus that the document exist, which it does once created; a read that found nothing + * carries no precondition, so the save creates the document and the next one overwrites it. + * + * The stamp arrives on a client-supplied url, so a value that is not one of the three forms makes + * the whole stamp unusable rather than being passed to a store. * * @param {string} [value] the raw param value * @returns {{kind: string, condition?: Object}|undefined} undefined when there is no stamp to - * trust, which leaves the fresh probe to decide on its own + * trust, which leaves the fresh lookup to decide on its own */ export function parseSourceStamp(value) { - if (!value) return undefined; if (value === LEGACY_STAMP) return { kind: LEGACY, condition: undefined }; if (value === SOURCE_BUS_STAMP) return { kind: SOURCE_BUS, condition: { 'If-Match': '*' } }; - - const [store, ...rest] = value.split('.'); - const etag = rest.join('.'); - if (store !== SOURCE_BUS_STAMP || !etag) return undefined; - if (etag === NEW_DOCUMENT) return { kind: SOURCE_BUS, condition: { 'If-None-Match': '*' } }; - if (!ETAG.test(etag)) return undefined; - return { kind: SOURCE_BUS, condition: { 'If-Match': `"${etag}"` } }; + if (value === `${SOURCE_BUS_STAMP}.${NEW_DOCUMENT}`) { + return { kind: SOURCE_BUS, condition: undefined }; + } + return undefined; } diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index b564e9c3..34990379 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -210,13 +210,13 @@ describe('reading with the content source resolved', () => { }); describe('the stamp a read leaves for its write', () => { - it('carries the source-bus etag it read', async () => { + it('names the source bus when the read found a document', async () => { const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.deepStrictEqual(seen.stamps, ['sb.busetag']); + assert.deepStrictEqual(seen.stamps, ['sb']); }); it('says the source-bus document is new when the read found nothing', async () => { @@ -231,10 +231,12 @@ describe('reading with the content source resolved', () => { assert.deepStrictEqual(seen.stamps, ['sb.new']); }); - it('says only the store when a source-bus read carried no etag', async () => { + // one page load produces many saves against the same stamp, so pinning the version would + // land the first and refuse the rest + it('carries no version, so it holds for every save in the session', async () => { const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE, - bus: () => new Response('x', { status: 200 }), + bus: () => new Response('x', { status: 200, headers: { etag: '"deadbeefcafe"' } }), }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); @@ -252,6 +254,64 @@ describe('reading with the content source resolved', () => { assert.deepStrictEqual(seen.stamps, ['da']); }); + // every other test here mocks applyUEInstrumentation, which leaves the one line that carries + // the stamp from daSourceGet into the emitted meta tag unguarded. These two run the real + // src/ue/ue.js and assert on the served body instead. + describe('as it reaches the served page', () => { + const servedBy = async (source) => { + const seen = { bus: [], legacy: [] }; + globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + if (request.url.startsWith('https://api.aem.live/')) { + seen.bus.push(request.url); + return new Response('

x

', { status: 200, headers: { etag: '"busetag"' } }); + } + // component-definition and friends, fetched by the real instrumentation. A missing one + // is what a site without them answers, and getUEConfig degrades to undefined. + return new Response('not found', { status: 404 }); + }; + const env = { + DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', + UE_HOST: 'ue.da.live', + daadmin: { + fetch: async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + seen.legacy.push(request.url); + return new Response('

x

', { status: 200 }); + }, + }, + }; + const { daSourceGet } = await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => source, + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, + '../../src/utils/aemCtx.js': { + getAemCtx: () => ({ ueHostname: 'ue.da.live', previewUrl: 'https://p.example' }), + getAEMHtml: async () => '', + }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + return res.text(); + }; + + it('lands on the connection uri for a source-bus read', async () => { + const html = await servedBy(BUS_SOURCE); + + assert.match(html, /urn:adobe:aue:system:ab" content="da:[^"]*\?ab-src=sb"/); + }); + + it('lands on the connection uri for a legacy read', async () => { + const html = await servedBy(LEGACY_SOURCE); + + assert.match(html, /urn:adobe:aue:system:ab" content="da:[^"]*\?ab-src=da"/); + }); + }); + it('is not applied outside UE, where nothing posts back', async () => { const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); const req = authedReq('https://main--site--org.preview.da.live/folder/content'); diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 866556b2..04f93f66 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -182,23 +182,28 @@ describe('a UE session, which saves many times against one page load', () => { describe('writing with the content source resolved', () => { describe('a stamped source-bus save', () => { it('goes to the source bus', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); assert.strictEqual(seen.legacy.length, 0); assert.strictEqual(seen.bus.length, 1); assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); }); - // this is the whole point: the etag came from the read, so a write conditioned on it cannot - // land on a document the author never saw - it('carries the etag the read returned as an If-Match', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + it('asks that the document still exist', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); - assert.strictEqual(seen.bus[0].headers.get('If-Match'), '"busetag"'); + assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); + }); + + it('pins no version, so the next save in the session is not refused', async () => { + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); + + assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); + assert.strictEqual(seen.bus[0].headers.get('If-None-Match'), null); }); it('sends the document as the raw body the source bus parses', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); assert.strictEqual(seen.bus[0].body, DOC); assert.strictEqual(seen.bus[0].contentType, 'text/html'); @@ -206,36 +211,30 @@ describe('writing with the content source resolved', () => { // helix-api-service parses no form data; the envelope would be stored as the document text it('never wraps the body in a multipart envelope', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.busetag`); + const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); assert.ok(!seen.bus[0].body.includes('Content-Disposition'), seen.bus[0].body); assert.ok(!/boundary/i.test(seen.bus[0].contentType ?? ''), seen.bus[0].contentType); }); it('passes the store answer back, so a 412 reaches the editor', async () => { - const { res } = await post({ source: BUS_SOURCE, status: 412 }, `${AT}?ab-src=sb.busetag`); + const { res } = await post({ source: BUS_SOURCE, status: 412 }, `${AT}?ab-src=sb`); assert.strictEqual(res.status, 412); }); }); describe('a save of a page the read did not find', () => { - it('may only create, so an existing page is refused by the store', async () => { + // If-None-Match: * would create on the first save and refuse every one after it + it('carries no precondition, so it creates and then overwrites', async () => { const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.new`); - assert.strictEqual(seen.bus[0].headers.get('If-None-Match'), '*'); + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].headers.get('If-None-Match'), null); assert.strictEqual(seen.bus[0].headers.get('If-Match'), null); }); }); - describe('a source-bus save whose read carried no etag', () => { - it('may overwrite but not create', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); - - assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); - }); - }); - describe('a stamped legacy save', () => { it('goes to da-admin as a data form part', async () => { const { seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=da`); @@ -267,7 +266,7 @@ describe('writing with the content source resolved', () => { }); it('refuses a source-bus-stamped save to a site now on da-admin', async () => { - const { res, seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=sb.busetag`); + const { res, seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=sb`); assert.strictEqual(res.status, 409); assert.strictEqual(seen.bus.length + seen.legacy.length, 0); @@ -303,7 +302,8 @@ describe('writing with the content source resolved', () => { describe('a save carrying a stamp that cannot be trusted', () => { [ ['an unknown store', 'gcs.abc'], - ['an etag with a quote in it', 'sb.a"b'], + ['a version pin, which this no longer emits', 'sb.9e8311043aab12b1'], + ['a value with a quote in it', 'sb.a"b'], ['a header injection attempt', 'sb.abc%0d%0aX-Evil:%201'], ['an empty value', ''], ].forEach(([what, value]) => { @@ -324,7 +324,7 @@ describe('writing with the content source resolved', () => { describe('when the content source could not be resolved', () => { it('refuses with 503 and touches neither store', async () => { - const { res, seen } = await post({ source: UNKNOWN_SOURCE }, `${AT}?ab-src=sb.busetag`); + const { res, seen } = await post({ source: UNKNOWN_SOURCE }, `${AT}?ab-src=sb`); assert.strictEqual(res.status, 503); assert.strictEqual(seen.bus.length + seen.legacy.length, 0); @@ -347,7 +347,7 @@ describe('writing with the content source resolved', () => { it('strips the UE data attributes before the store sees them', async () => { const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); const req = uePost( - `${AT}?ab-src=sb.busetag`, + `${AT}?ab-src=sb`, '

text

', ); @@ -371,7 +371,7 @@ describe('writing with the content source resolved', () => { it('keeps the case the source bus stores it under', async () => { const { seen } = await post( { source: BUS_SOURCE }, - 'https://main--site--org.ue.da.live/Folder/Content?ab-src=sb.busetag', + 'https://main--site--org.ue.da.live/Folder/Content?ab-src=sb', ); assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Folder/content.html'); diff --git a/test/utils/source-stamp.test.js b/test/utils/source-stamp.test.js index b7e79a0b..7fc236b5 100644 --- a/test/utils/source-stamp.test.js +++ b/test/utils/source-stamp.test.js @@ -26,59 +26,47 @@ describe('formatSourceStamp', () => { assert.strictEqual(SOURCE_STAMP_PARAM, 'ab-src'); }); - it('stamps a source-bus read with the etag it read', () => { - assert.strictEqual(formatSourceStamp(bus, '"9e8311043aab12b1"'), 'sb.9e8311043aab12b1'); + it('stamps a source-bus read that found a document', () => { + assert.strictEqual(formatSourceStamp(bus, true), 'sb'); }); - it('strips the quotes, so the stamp needs no url encoding', () => { - assert.doesNotMatch(formatSourceStamp(bus, '"abc123"'), /["%]/); + it('stamps a source-bus read that found nothing', () => { + assert.strictEqual(formatSourceStamp(bus, false), 'sb.new'); }); - it('unwraps a weak etag', () => { - assert.strictEqual(formatSourceStamp(bus, 'W/"abc123"'), 'sb.abc123'); + // one page load produces many saves against the same stamp, so a version in it would land the + // first save and refuse the rest + it('carries no version, whatever etag the read returned', () => { + assert.doesNotMatch(formatSourceStamp(bus, true), /[0-9a-f]{8}/); }); - it('keeps a multipart etag suffix', () => { - assert.strictEqual(formatSourceStamp(bus, '"abc123-7"'), 'sb.abc123-7'); + it('needs no url encoding', () => { + ['sb', 'sb.new', 'da'].forEach((v) => assert.strictEqual(encodeURIComponent(v), v)); }); - it('stamps a source-bus read that found nothing, so the write can only create', () => { - assert.strictEqual(formatSourceStamp(bus, undefined), 'sb.new'); - }); - - it('stamps a source-bus read with no etag as a bare source-bus read', () => { - assert.strictEqual(formatSourceStamp(bus, null, true), 'sb'); - }); - - it('falls back to a bare source-bus read for an etag it cannot put in a url', () => { - assert.strictEqual(formatSourceStamp(bus, '"has spaces and /"', true), 'sb'); - }); - - it('stamps a legacy read, which has no etag to carry', () => { - assert.strictEqual(formatSourceStamp(legacy, undefined), 'da'); + it('stamps a legacy read, which has no version to carry either', () => { + assert.strictEqual(formatSourceStamp(legacy, false), 'da'); }); it('stamps a legacy read the same whether or not the document was found', () => { - assert.strictEqual(formatSourceStamp(legacy, undefined, true), 'da'); + assert.strictEqual(formatSourceStamp(legacy, true), 'da'); }); }); describe('parseSourceStamp', () => { describe('a source-bus stamp', () => { it('reads the store back', () => { - assert.strictEqual(parseSourceStamp('sb.abc123').kind, SOURCE_BUS); + assert.strictEqual(parseSourceStamp('sb').kind, SOURCE_BUS); + assert.strictEqual(parseSourceStamp('sb.new').kind, SOURCE_BUS); }); - it('turns the etag into an If-Match, so a changed page is refused', () => { - assert.deepStrictEqual(parseSourceStamp('sb.abc123').condition, { 'If-Match': '"abc123"' }); - }); - - it('turns a new-page stamp into If-None-Match, so an existing page is refused', () => { - assert.deepStrictEqual(parseSourceStamp('sb.new').condition, { 'If-None-Match': '*' }); + it('asks that the document still exist, which holds for every save in a session', () => { + assert.deepStrictEqual(parseSourceStamp('sb').condition, { 'If-Match': '*' }); }); - it('turns a bare stamp into If-Match: *, so it can overwrite but not create', () => { - assert.deepStrictEqual(parseSourceStamp('sb').condition, { 'If-Match': '*' }); + // If-None-Match: * would refuse every save after the one that created the document + it('carries no precondition when the read found nothing, so the save can create it', () => { + assert.strictEqual(parseSourceStamp('sb.new').condition, undefined); }); }); @@ -100,10 +88,12 @@ describe('parseSourceStamp', () => { ['an empty stamp', ''], ['an unknown store', 'gcs.abc'], ['a stamp shaped like a path', 'sb/abc'], - ['an etag with a url-unsafe character', 'sb.a"b'], - ['an etag with a slash', 'sb.a/b'], - ['a stamp with an empty etag', 'sb.'], - ['a stamp trying to inject a header', 'sb.abc\r\nX-Evil: 1'], + ['a version pin, which this no longer emits', 'sb.9e8311043aab12b1'], + ['a url-unsafe character', 'sb.a"b'], + ['a slash', 'sb.a/b'], + ['a trailing dot', 'sb.'], + ['a header injection attempt', 'sb.abc\r\nX-Evil: 1'], + ['a case variation', 'SB'], ].forEach(([what, value]) => { it(`is not trusted: ${what}`, () => { assert.strictEqual(parseSourceStamp(value), undefined); @@ -112,19 +102,12 @@ describe('parseSourceStamp', () => { }); describe('round trip', () => { - it('parses back what a source-bus read stamped', () => { - const stamp = formatSourceStamp(bus, '"9e8311043aab12b156073d30f8bb3710"'); - - assert.deepStrictEqual(parseSourceStamp(stamp), { - kind: SOURCE_BUS, - condition: { 'If-Match': '"9e8311043aab12b156073d30f8bb3710"' }, - }); - }); + [[bus, true], [bus, false], [legacy, true], [legacy, false]].forEach(([source, found]) => { + it(`parses back what a ${source.kind} read stamped, found=${found}`, () => { + const parsed = parseSourceStamp(formatSourceStamp(source, found)); - it('parses back what a legacy read stamped', () => { - assert.deepStrictEqual(parseSourceStamp(formatSourceStamp(legacy)), { - kind: LEGACY, - condition: undefined, + assert.ok(parsed, 'a stamp this code emits must parse back'); + assert.strictEqual(parsed.kind, source.kind); }); }); }); From 28146a511478f314fe71ac244752ade4d7fbc5f8 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 06:07:17 +0200 Subject: [PATCH 26/48] test: pin reusing a lookup across a page's image reads a previewed page is one worker request per image, each looking the store up again at ~460ms. writes never reuse, and an unresolved answer is never kept. --- test/storage/content-source.test.js | 88 +++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index e3df62f2..4a025a31 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -222,6 +222,94 @@ describe('resolveContentSource', () => { }); }); + describe('reusing an answer within a page load', () => { + // one previewed page is many worker requests: the document, then one per relative image src, + // each of which would look the store up again. Measured live on 2026-08-03: the sidekick + // config is `cache-control: no-store` and costs ~460ms, so 8 identical lookups spend 3.8s of + // origin time and add that to every image. + it('asks once for a burst of reads of the same site', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx(), { reuse: true }); + await resolveContentSource(env, daCtx(), { reuse: true }); + await resolveContentSource(env, daCtx(), { reuse: true }); + + assert.strictEqual(calls.length, 1); + }); + + it('gives the same answer each time', async () => { + stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source')); + + const first = await resolveContentSource(env, daCtx(), { reuse: true }); + const second = await resolveContentSource(env, daCtx(), { reuse: true }); + + assert.deepStrictEqual(second, first); + }); + + it('asks again for a different site', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx(), { reuse: true }); + await resolveContentSource(env, daCtx({ site: 'other' }), { reuse: true }); + + assert.strictEqual(calls.length, 2); + }); + + it('asks again for a different ref of the same site', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx(), { reuse: true }); + await resolveContentSource(env, daCtx({ ref: 'branch' }), { reuse: true }); + + assert.strictEqual(calls.length, 2); + }); + + // a write is the one operation a wrong store cannot be walked back from, so it always asks + it('does not reuse an answer unless asked to', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx(), { reuse: true }); + await resolveContentSource(env, daCtx()); + + assert.strictEqual(calls.length, 2); + }); + + it('never lets a write read a stored answer', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx()); + await resolveContentSource(env, daCtx()); + + assert.strictEqual(calls.length, 2); + }); + + // an outage that stuck would outlast itself + it('never stores an answer it could not give', async () => { + stubFetch(() => new Response('', { status: 503 })); + + await resolveContentSource(env, daCtx({ site: 'flaky' }), { reuse: true }); + await resolveContentSource(env, daCtx({ site: 'flaky' }), { reuse: true }); + + assert.strictEqual(calls.length, 2); + }); + + it('picks up a recovery on the next read', async () => { + let attempt = 0; + stubFetch(() => { + attempt += 1; + return attempt === 1 + ? new Response('', { status: 503 }) + : sidekick('https://api.aem.live/org/sites/recovering/source'); + }); + + const down = await resolveContentSource(env, daCtx({ site: 'recovering' }), { reuse: true }); + const up = await resolveContentSource(env, daCtx({ site: 'recovering' }), { reuse: true }); + + assert.strictEqual(down.kind, 'unknown'); + assert.strictEqual(up.kind, 'sourcebus'); + }); + }); + describe('the admin host', () => { // the caller turns unknown into a 503 it can return; a throw here escapes into // withCorsHeaders, which reads response.headers and throws again on undefined From 23148d1d6605caaf3658cca601091d8d3d0f5f54 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 06:08:27 +0200 Subject: [PATCH 27/48] perf: reuse a store lookup across one page's image reads a previewed page is one worker request per relative image src, each looking the store up again. the sidekick config is no-store and ~460ms, so 8 identical lookups spend 3.8s of origin time. reads reuse an answer for 10s; a write always asks, and an unresolved answer is never kept. --- src/routes/da-admin.js | 6 ++-- src/storage/content-source.js | 33 +++++++++++++++---- test/storage/content-source.test.js | 51 ++++++++++++++--------------- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index 9847bb88..bfdf22c9 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -107,7 +107,7 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing - const source = await resolveContentSource(env, daCtx); + const source = await resolveContentSource(env, daCtx, { reuse: true }); if (source.kind === UNKNOWN) { console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); @@ -123,7 +123,7 @@ export async function daSourceGet({ req, env, daCtx }) { const aemCtx = getAemCtx(env, daCtx); const [headHtml, source] = await Promise.all([ getAEMHtml(aemCtx, '/head.html'), - resolveContentSource(env, daCtx), + resolveContentSource(env, daCtx, { reuse: true }), ]); if (!headHtml) { // quick-edit still needs a working shell (with the import map) so the editor @@ -201,7 +201,7 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const source = await resolveContentSource(env, daCtx); + const source = await resolveContentSource(env, daCtx, { reuse: true }); if (source.kind === UNKNOWN) { console.warn(`503 HEAD ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return head503(); diff --git a/src/storage/content-source.js b/src/storage/content-source.js index 2aee7dd9..3e1f7540 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -13,11 +13,19 @@ const SOURCE_BUS_PREFIX = 'https://api.aem.live/'; const LEGACY_PREFIX = 'https://content.da.live/'; const TIMEOUT_MS = 5 * 1000; +const REUSE_MS = 10 * 1000; export const SOURCE_BUS = 'sourcebus'; export const LEGACY = 'legacy'; export const UNKNOWN = 'unknown'; +/** + * Answers a read has already paid for, so the images on one page do not each pay again. Only ever + * read on the read path, and only ever holds an answer that was given, so an outage does not + * outlast itself. + */ +const recent = new Map(); + function unknown(org, site, reason) { console.warn(`[source] ${org}/${site} unknown: ${reason}`); return { kind: UNKNOWN, reason }; @@ -34,10 +42,14 @@ function unknown(org, site, reason) { * * @param {Object} env worker env, `HLX_ADMIN` is the admin host * @param {Object} daCtx + * @param {Object} [opts] + * @param {boolean} [opts.reuse] reuse an answer given for this site in the last few seconds. Set + * on reads, where one previewed page is a request per image; never on a write, which is the one + * operation a wrong store cannot be walked back from. * @returns {Promise<{kind: string, base?: string, reason?: string}>} `sourcebus` with the store * base url, `legacy`, or `unknown` with the reason it could not be answered */ -export default async function resolveContentSource(env, daCtx) { +export default async function resolveContentSource(env, daCtx, { reuse = false } = {}) { const { org, site, ref, authToken, } = daCtx; @@ -48,6 +60,12 @@ export default async function resolveContentSource(env, daCtx) { return unknown(org, site, 'no org or site in the request'); } + const key = `${org}/${site}/${ref}`; + if (reuse) { + const held = recent.get(key); + if (held && Date.now() - held.at < REUSE_MS) return held.source; + } + let url; try { url = new URL(`/sidekick/${org}/${site}/${ref}/config.json`, env.HLX_ADMIN); @@ -80,11 +98,14 @@ export default async function resolveContentSource(env, daCtx) { if (typeof sourceUrl !== 'string') { return unknown(org, site, `${url} named no content source`); } + let source; if (sourceUrl.startsWith(SOURCE_BUS_PREFIX)) { - return { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; - } - if (sourceUrl.startsWith(LEGACY_PREFIX)) { - return { kind: LEGACY }; + source = { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; + } else if (sourceUrl.startsWith(LEGACY_PREFIX)) { + source = { kind: LEGACY }; + } else { + return unknown(org, site, `content source ${sourceUrl} is neither store`); } - return unknown(org, site, `content source ${sourceUrl} is neither store`); + recent.set(key, { source, at: Date.now() }); + return source; } diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index 4a025a31..44e42dbb 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -227,21 +227,25 @@ describe('resolveContentSource', () => { // each of which would look the store up again. Measured live on 2026-08-03: the sidekick // config is `cache-control: no-store` and costs ~460ms, so 8 identical lookups spend 3.8s of // origin time and add that to every image. + // + // What is held is per isolate and shared across requests, so each test here uses its own site. it('asks once for a burst of reads of the same site', async () => { stubFetch(legacyBody); + const ctx = daCtx({ site: 'burst' }); - await resolveContentSource(env, daCtx(), { reuse: true }); - await resolveContentSource(env, daCtx(), { reuse: true }); - await resolveContentSource(env, daCtx(), { reuse: true }); + await resolveContentSource(env, ctx, { reuse: true }); + await resolveContentSource(env, ctx, { reuse: true }); + await resolveContentSource(env, ctx, { reuse: true }); assert.strictEqual(calls.length, 1); }); it('gives the same answer each time', async () => { - stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source')); + stubFetch(() => sidekick('https://api.aem.live/org/sites/same/source')); + const ctx = daCtx({ site: 'same' }); - const first = await resolveContentSource(env, daCtx(), { reuse: true }); - const second = await resolveContentSource(env, daCtx(), { reuse: true }); + const first = await resolveContentSource(env, ctx, { reuse: true }); + const second = await resolveContentSource(env, ctx, { reuse: true }); assert.deepStrictEqual(second, first); }); @@ -249,8 +253,8 @@ describe('resolveContentSource', () => { it('asks again for a different site', async () => { stubFetch(legacyBody); - await resolveContentSource(env, daCtx(), { reuse: true }); - await resolveContentSource(env, daCtx({ site: 'other' }), { reuse: true }); + await resolveContentSource(env, daCtx({ site: 'one' }), { reuse: true }); + await resolveContentSource(env, daCtx({ site: 'two' }), { reuse: true }); assert.strictEqual(calls.length, 2); }); @@ -258,8 +262,8 @@ describe('resolveContentSource', () => { it('asks again for a different ref of the same site', async () => { stubFetch(legacyBody); - await resolveContentSource(env, daCtx(), { reuse: true }); - await resolveContentSource(env, daCtx({ ref: 'branch' }), { reuse: true }); + await resolveContentSource(env, daCtx({ site: 'refs' }), { reuse: true }); + await resolveContentSource(env, daCtx({ site: 'refs', ref: 'branch' }), { reuse: true }); assert.strictEqual(calls.length, 2); }); @@ -267,28 +271,22 @@ describe('resolveContentSource', () => { // a write is the one operation a wrong store cannot be walked back from, so it always asks it('does not reuse an answer unless asked to', async () => { stubFetch(legacyBody); + const ctx = daCtx({ site: 'writes' }); - await resolveContentSource(env, daCtx(), { reuse: true }); - await resolveContentSource(env, daCtx()); - - assert.strictEqual(calls.length, 2); - }); + await resolveContentSource(env, ctx, { reuse: true }); + await resolveContentSource(env, ctx); + await resolveContentSource(env, ctx); - it('never lets a write read a stored answer', async () => { - stubFetch(legacyBody); - - await resolveContentSource(env, daCtx()); - await resolveContentSource(env, daCtx()); - - assert.strictEqual(calls.length, 2); + assert.strictEqual(calls.length, 3); }); // an outage that stuck would outlast itself it('never stores an answer it could not give', async () => { stubFetch(() => new Response('', { status: 503 })); + const ctx = daCtx({ site: 'flaky' }); - await resolveContentSource(env, daCtx({ site: 'flaky' }), { reuse: true }); - await resolveContentSource(env, daCtx({ site: 'flaky' }), { reuse: true }); + await resolveContentSource(env, ctx, { reuse: true }); + await resolveContentSource(env, ctx, { reuse: true }); assert.strictEqual(calls.length, 2); }); @@ -301,9 +299,10 @@ describe('resolveContentSource', () => { ? new Response('', { status: 503 }) : sidekick('https://api.aem.live/org/sites/recovering/source'); }); + const ctx = daCtx({ site: 'recovering' }); - const down = await resolveContentSource(env, daCtx({ site: 'recovering' }), { reuse: true }); - const up = await resolveContentSource(env, daCtx({ site: 'recovering' }), { reuse: true }); + const down = await resolveContentSource(env, ctx, { reuse: true }); + const up = await resolveContentSource(env, ctx, { reuse: true }); assert.strictEqual(down.kind, 'unknown'); assert.strictEqual(up.kind, 'sourcebus'); From 5b098f4da227be03d7426d34f87ec22b0b59537b Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 06:10:09 +0200 Subject: [PATCH 28/48] test: pin that a write never reuses a stored lookup a stale answer would let the 409 miss a site that moved stores inside the reuse window, which is the one case that overwrites a live page. found by a surviving mutation. --- test/routes/source-write.test.js | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 04f93f66..84902b37 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -322,6 +322,84 @@ describe('writing with the content source resolved', () => { }); }); + describe('how fresh the lookup on a write is', () => { + // reads reuse an answer for a few seconds so one page's images do not each pay for it. A + // write must not: a stale answer would let the 409 miss a site that moved stores in that + // window, which is the one case that overwrites a live page. + const optsSeenBy = async (drive) => { + const opts = []; + globalThis.fetch = async () => new Response('', { status: 201 }); + const env = { + DA_ADMIN: 'https://admin.da.live', + HLX_ADMIN: 'https://admin.hlx.page', + daadmin: { fetch: async () => new Response('', { status: 201 }) }, + }; + const mod = await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async (e, ctx, o) => { + opts.push(o); + return BUS_SOURCE; + }, + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, + '../../src/utils/aemCtx.js': { + getAemCtx: () => ({}), + getAEMHtml: async () => '', + }, + '../../src/render/compose.js': { + composeHtml: async () => ({}), + serializeHtml: () => 'c', + }, + '../../src/ue/ue.js': { applyUEInstrumentation: async () => {} }, + '../../src/storage/config.js': { + getSiteConfig: async () => { throw new Error('none'); }, + }, + }); + await drive(mod, env); + return opts; + }; + + it('asks for a fresh answer on a write', async () => { + const opts = await optsSeenBy(async (mod, env) => { + const request = uePost(`${AT}?ab-src=sb`); + await mod.daSourcePost({ req: request, env, daCtx: getDaCtx(request) }); + }); + + assert.strictEqual(opts.length, 1); + assert.notStrictEqual(opts[0]?.reuse, true); + }); + + it('lets a read reuse one', async () => { + const opts = await optsSeenBy(async (mod, env) => { + const request = new Request(AT, { headers: { Authorization: 'Bearer t' } }); + await mod.daSourceGet({ req: request, env, daCtx: getDaCtx(request) }); + }); + + assert.strictEqual(opts[0]?.reuse, true); + }); + + it('lets a HEAD reuse one', async () => { + const opts = await optsSeenBy(async (mod, env) => { + const request = new Request(AT, { headers: { Authorization: 'Bearer t' } }); + await mod.daSourceHead({ env, daCtx: getDaCtx(request) }); + }); + + assert.strictEqual(opts[0]?.reuse, true); + }); + + it('lets a non-html read reuse one', async () => { + const opts = await optsSeenBy(async (mod, env) => { + const url = 'https://main--site--org.ue.da.live/photo.png'; + const request = new Request(url, { headers: { Authorization: 'Bearer t' } }); + await mod.daSourceGet({ req: request, env, daCtx: getDaCtx(request) }); + }); + + assert.strictEqual(opts[0]?.reuse, true); + }); + }); + describe('when the content source could not be resolved', () => { it('refuses with 503 and touches neither store', async () => { const { res, seen } = await post({ source: UNKNOWN_SOURCE }, `${AT}?ab-src=sb`); From 2ef64402ba28a9cbb39d95ed94a1d98a439c4265 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 09:36:48 +0200 Subject: [PATCH 29/48] refactor: ask the AEM API for the store, and drop what guarded nothing the lookup moves to api.aem.live/{org}/sites/{site}/sidekick behind AEM_API. it needs code:read, which every role that can save already has, and it answers for legacy sites too since both stores read the same config service. gone with it: the connection-uri marker and the 409 it fed, which caught only a migration mid-session and those are managed; the write preconditions, which no read could refresh; and the 10s lookup reuse, since nothing else on this path caches and the config owner sends no-cache upstream and no-store down. --- src/responses/index.js | 4 - src/routes/da-admin.js | 41 +-- src/storage/content-source.js | 66 ++--- src/storage/store.js | 8 +- src/ue/scaffold.js | 8 +- src/ue/ue.js | 6 +- src/utils/constants.js | 2 - src/utils/daCtx.js | 6 - src/utils/source-stamp.js | 65 ----- test/routes/da-admin.test.js | 4 +- test/routes/source-read.test.js | 140 ++++------ test/routes/source-write.test.js | 354 +++++--------------------- test/storage/content-source.test.js | 135 +++------- test/storage/store.test.js | 26 +- test/ue/source-stamp-scaffold.test.js | 75 ------ test/utils/source-stamp.test.js | 114 --------- wrangler.toml | 6 +- 17 files changed, 203 insertions(+), 857 deletions(-) delete mode 100644 src/utils/source-stamp.js delete mode 100644 test/ue/source-stamp-scaffold.test.js delete mode 100644 test/utils/source-stamp.test.js diff --git a/src/responses/index.js b/src/responses/index.js index 8617cfa9..e1639e8d 100644 --- a/src/responses/index.js +++ b/src/responses/index.js @@ -60,10 +60,6 @@ export function get503(message = '') { // a refused write is never rendered. The Universal Editor Service embeds the body verbatim in // its problem+json error string, so plain text is what an author is shown. -export function post409(message = '') { - return daResp({ body: message, status: 409, contentType: 'text/plain; charset=utf-8' }); -} - export function post503(message = '') { return daResp({ body: message, diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index bfdf22c9..bc346cb4 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -22,20 +22,18 @@ import { applyQuickEditToDocument, buildQuickEditCookie, buildQuickEditNotFoundResponse, } from '../utils/quick-edit.js'; import { - daResp, get401, get404, get415, get503, head401, head503, post409, post503, + daResp, get401, get404, get415, get503, head401, head503, post503, } from '../responses/index.js'; import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, - SOURCE_MOVED_MESSAGE, SOURCE_UNRESOLVED_HTML_MESSAGE, SOURCE_UNRESOLVED_MESSAGE, UNAUTHORIZED_HTML_MESSAGE, } from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; -import resolveContentSource, { SOURCE_BUS, UNKNOWN } from '../storage/content-source.js'; +import resolveContentSource, { UNKNOWN } from '../storage/content-source.js'; import getStore from '../storage/store.js'; -import { formatSourceStamp, parseSourceStamp } from '../utils/source-stamp.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; const HTML_POST_TYPE = 'text/html'; @@ -107,7 +105,7 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing - const source = await resolveContentSource(env, daCtx, { reuse: true }); + const source = await resolveContentSource(env, daCtx); if (source.kind === UNKNOWN) { console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); @@ -123,7 +121,7 @@ export async function daSourceGet({ req, env, daCtx }) { const aemCtx = getAemCtx(env, daCtx); const [headHtml, source] = await Promise.all([ getAEMHtml(aemCtx, '/head.html'), - resolveContentSource(env, daCtx, { reuse: true }), + resolveContentSource(env, daCtx), ]); if (!headHtml) { // quick-edit still needs a working shell (with the import map) so the editor @@ -156,9 +154,8 @@ export async function daSourceGet({ req, env, daCtx }) { return sourceResp; } - const found = sourceResp.status === 200; // use the stored content when available, otherwise fall back to a template - const bodyHtml = found + const bodyHtml = sourceResp.status === 200 ? await sourceResp.text() : await getPageTemplate(env, daCtx, aemCtx, headHtml); @@ -175,9 +172,7 @@ export async function daSourceGet({ req, env, daCtx }) { extraHeaders.push(['Set-Cookie', buildQuickEditCookie(entryPath)]); } } else if (isUE) { - // UE is the only client that posts back, so it is the only one that needs the stamp - const stamp = formatSourceStamp(source, found); - await applyUEInstrumentation(documentTree, daCtx, aemCtx, stamp); + await applyUEInstrumentation(documentTree, daCtx, aemCtx); } const body = serializeHtml(documentTree); @@ -201,7 +196,7 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const source = await resolveContentSource(env, daCtx, { reuse: true }); + const source = await resolveContentSource(env, daCtx); if (source.kind === UNKNOWN) { console.warn(`503 HEAD ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return head503(); @@ -256,29 +251,11 @@ export async function daSourcePost({ req, env, daCtx }) { return post503(SOURCE_UNRESOLVED_MESSAGE); } - // the read stamped the connection uri with the store it came from, and the Universal Editor - // Service posts back to that uri, so the two requests are linked. Where the stamp and a - // fresh lookup disagree, the site moved stores while the page was open: writing to the store - // the content came from orphans it, writing to the new one overwrites a page the author - // never saw, and neither is worth doing silently. - const stamp = parseSourceStamp(daCtx.sourceStamp); - if (stamp && stamp.kind !== source.kind) { - console.warn(`409 POST ${sourcePath}, read from ${stamp.kind} but the site is on ${source.kind}`); - return post409(SOURCE_MOVED_MESSAGE); - } - - // with no stamp there is no provenance, so a source-bus write may overwrite a page that - // exists but may not invent one. A stamp that carries no precondition is not the same thing: - // it says the read looked at the source bus and found nothing, so creating is what it asked - // for. - const noProvenance = source.kind === SOURCE_BUS ? { 'If-Match': '*' } : undefined; - const condition = stamp ? stamp.condition : noProvenance; - // the two stores take the document in different shapes, so the store builds its own request const store = getStore(env, daCtx, source); // eslint-disable-next-line no-param-reassign - req = new Request(store.url, store.writeInit(bodyContent, authToken, condition)); - console.log(`-> ${store.url.toString()}${condition ? ` ${JSON.stringify(condition)}` : ''}`); + req = new Request(store.url, store.writeInit(bodyContent, authToken)); + console.log(`-> ${store.url.toString()}`); const response = await store.fetch(req); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; diff --git a/src/storage/content-source.js b/src/storage/content-source.js index 3e1f7540..f674b050 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -10,67 +10,50 @@ * governing permissions and limitations under the License. */ -const SOURCE_BUS_PREFIX = 'https://api.aem.live/'; const LEGACY_PREFIX = 'https://content.da.live/'; const TIMEOUT_MS = 5 * 1000; -const REUSE_MS = 10 * 1000; export const SOURCE_BUS = 'sourcebus'; export const LEGACY = 'legacy'; export const UNKNOWN = 'unknown'; -/** - * Answers a read has already paid for, so the images on one page do not each pay again. Only ever - * read on the read path, and only ever holds an answer that was given, so an outage does not - * outlast itself. - */ -const recent = new Map(); - function unknown(org, site, reason) { console.warn(`[source] ${org}/${site} unknown: ${reason}`); return { kind: UNKNOWN, reason }; } /** - * Asks admin.hlx.page which store holds a site's content. + * Asks the AEM API which store holds a site's content. + * + * `GET {AEM_API}/{org}/sites/{site}/sidekick` returns the resolved content source url in its body + * and needs only `code:read`, the permission every authoring role already has. It answers for + * legacy sites too, because both stores read the same config service. A config that could not be + * resolved is a 404 rather than a wrong answer, which is what lets an unresolved source be + * refused instead of guessed at. * - * The sidekick config returns the resolved content source url in its body, and answers 404 when - * config resolution produced nothing (`if (config) { ... } return { status: 404 }` in - * helix-admin src/sidekick/handler.js). So a failure to resolve is reported as a failure. The - * `/ping` header cannot do that: it is absent both for a legacy site and for a source-bus site - * whose config could not be read, which is the case that reads past a page and writes over it. + * The prefix test is the same one the platform applies to itself: + * `helix-api-service/src/contentproxy/index.js` reads the source as the source bus when its url + * starts with the API host. * - * @param {Object} env worker env, `HLX_ADMIN` is the admin host + * @param {Object} env worker env, `AEM_API` is the API host and the source-bus prefix * @param {Object} daCtx - * @param {Object} [opts] - * @param {boolean} [opts.reuse] reuse an answer given for this site in the last few seconds. Set - * on reads, where one previewed page is a request per image; never on a write, which is the one - * operation a wrong store cannot be walked back from. * @returns {Promise<{kind: string, base?: string, reason?: string}>} `sourcebus` with the store * base url, `legacy`, or `unknown` with the reason it could not be answered */ -export default async function resolveContentSource(env, daCtx, { reuse = false } = {}) { - const { - org, site, ref, authToken, - } = daCtx; +export default async function resolveContentSource(env, daCtx) { + const { org, site, authToken } = daCtx; - // an unparseable hostname leaves org, site and ref all undefined together, and there is no - // site to ask about + // an unparseable hostname leaves org and site undefined, and there is no site to ask about if (!org || !site) { return unknown(org, site, 'no org or site in the request'); } - const key = `${org}/${site}/${ref}`; - if (reuse) { - const held = recent.get(key); - if (held && Date.now() - held.at < REUSE_MS) return held.source; - } - + const api = env.AEM_API?.replace(/\/$/, ''); let url; try { - url = new URL(`/sidekick/${org}/${site}/${ref}/config.json`, env.HLX_ADMIN); + url = new URL(`/${org}/sites/${site}/sidekick`, api); } catch (e) { - return unknown(org, site, `HLX_ADMIN is not a url: ${e.message}`); + return unknown(org, site, `AEM_API is not a url: ${e.message}`); } const headers = new Headers(); @@ -98,14 +81,11 @@ export default async function resolveContentSource(env, daCtx, { reuse = false } if (typeof sourceUrl !== 'string') { return unknown(org, site, `${url} named no content source`); } - let source; - if (sourceUrl.startsWith(SOURCE_BUS_PREFIX)) { - source = { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; - } else if (sourceUrl.startsWith(LEGACY_PREFIX)) { - source = { kind: LEGACY }; - } else { - return unknown(org, site, `content source ${sourceUrl} is neither store`); + if (sourceUrl.startsWith(`${api}/`)) { + return { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; + } + if (sourceUrl.startsWith(LEGACY_PREFIX)) { + return { kind: LEGACY }; } - recent.set(key, { source, at: Date.now() }); - return source; + return unknown(org, site, `content source ${sourceUrl} is neither store`); } diff --git a/src/storage/store.js b/src/storage/store.js index 93892b39..335d131e 100644 --- a/src/storage/store.js +++ b/src/storage/store.js @@ -52,10 +52,10 @@ export default function getStore(env, daCtx, source) { return { url: new URL(`${source.base}${sourceBusPath(daCtx)}`), fetch: (input, init) => fetch(input, init), - writeInit: (html, authToken, condition) => ({ + writeInit: (html, authToken) => ({ method: 'POST', body: html, - headers: { Authorization: authToken, 'Content-Type': 'text/html', ...condition }, + headers: { Authorization: authToken, 'Content-Type': 'text/html' }, }), }; } @@ -63,10 +63,10 @@ export default function getStore(env, daCtx, source) { return { url: new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN), fetch: (input, init) => env.daadmin.fetch(input, init), - writeInit: (html, authToken, condition) => { + writeInit: (html, authToken) => { const body = new FormData(); body.set('data', new Blob([html], { type: 'text/html' })); - return { method: 'POST', body, headers: { Authorization: authToken, ...condition } }; + return { method: 'POST', body, headers: { Authorization: authToken } }; }, }; } diff --git a/src/ue/scaffold.js b/src/ue/scaffold.js index e0618724..2d673abe 100644 --- a/src/ue/scaffold.js +++ b/src/ue/scaffold.js @@ -12,9 +12,8 @@ import { h } from 'hastscript'; import { withAemAuth } from '../utils/aemCtx.js'; -import { SOURCE_STAMP_PARAM } from '../utils/source-stamp.js'; -export function getUEHtmlHeadEntries(daCtx, aemCtx, sourceStamp) { +export function getUEHtmlHeadEntries(daCtx, aemCtx) { const { org, site, @@ -29,12 +28,9 @@ export function getUEHtmlHeadEntries(daCtx, aemCtx, sourceStamp) { let finalUeService = ueService; const children = []; - // the Universal Editor Service reads this uri and posts back to it verbatim, so the stamp on - // it is what tells a save which store the content it is saving was read from - const stamp = sourceStamp ? `?${SOURCE_STAMP_PARAM}=${sourceStamp}` : ''; children.push(h('meta', { name: 'urn:adobe:aue:system:ab', - content: isLocal ? `da:https://${ueHostname}/${org}/${site}${path}${stamp}` : `da:https://${ref}--${site}--${org}.${ueHostname}${path}${stamp}`, + content: isLocal ? `da:https://${ueHostname}/${org}/${site}${path}` : `da:https://${ref}--${site}--${org}.${ueHostname}${path}`, })); if (ueServiceParam && ueServiceParam === 'local') { diff --git a/src/ue/ue.js b/src/ue/ue.js index d9a3961b..e03b5118 100644 --- a/src/ue/ue.js +++ b/src/ue/ue.js @@ -21,13 +21,11 @@ import { injectUEAttributes } from './attributes.js'; * @param {import('hast').Root} documentTree - The composed document tree (mutated in place). * @param {Object} daCtx - The Dark Alley context object. * @param {Object} aemCtx - The AEM context object. - * @param {string} [sourceStamp] - The store and version this page was read from, carried on the - * connection uri so the save that follows lands in the same place. */ -export async function applyUEInstrumentation(documentTree, daCtx, aemCtx, sourceStamp) { +export async function applyUEInstrumentation(documentTree, daCtx, aemCtx) { // add UE head script and meta tags const headNode = select('head', documentTree); - headNode.children.push(...getUEHtmlHeadEntries(daCtx, aemCtx, sourceStamp)); + headNode.children.push(...getUEHtmlHeadEntries(daCtx, aemCtx)); // add data attributes for UE to the body const bodyNode = select('body', documentTree); diff --git a/src/utils/constants.js b/src/utils/constants.js index 36b55f96..81343083 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -55,6 +55,4 @@ export const SOURCE_UNRESOLVED_HTML_MESSAGE = '

503: Content sour export const SOURCE_UNRESOLVED_MESSAGE = 'The store that holds this document could not be determined, so the write was refused rather than sent to the wrong one. Please retry.'; -export const SOURCE_MOVED_MESSAGE = 'This document moved to a different content store while it was open. Reload the page to pick up the new one; saving now would write to the store it left.'; - export const DEFAULT_UNAUTHORIZED_HTML_MESSAGE = '

401: Unauthorized

'; diff --git a/src/utils/daCtx.js b/src/utils/daCtx.js index 12e4d128..42076717 100644 --- a/src/utils/daCtx.js +++ b/src/utils/daCtx.js @@ -9,7 +9,6 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ -import { SOURCE_STAMP_PARAM } from './source-stamp.js'; function getRefSiteOrgPath(hostname, pathname) { if (hostname === 'localhost') { @@ -114,11 +113,6 @@ export function getDaCtx(req) { const ueService = searchParams.get('ue-service'); if (ueService !== null) daCtx.ueService = ueService; - // the store a UE read came from, stamped onto the connection uri that the Universal Editor - // Service posts back to. Raw here; only source-stamp.js decides what it may be trusted to say. - const sourceStamp = searchParams.get(SOURCE_STAMP_PARAM); - if (sourceStamp !== null) daCtx.sourceStamp = sourceStamp; - daCtx.authToken = getAuthToken(req); daCtx.siteToken = getSiteToken(req); return daCtx; diff --git a/src/utils/source-stamp.js b/src/utils/source-stamp.js deleted file mode 100644 index 2bc70c6f..00000000 --- a/src/utils/source-stamp.js +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2026 Adobe. All rights reserved. - * This file is licensed to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS - * OF ANY KIND, either express or implied. See the License for the specific language - * governing permissions and limitations under the License. - */ -import { LEGACY, SOURCE_BUS } from '../storage/content-source.js'; - -/** - * The query param that carries the stamp on the Universal Editor connection uri. - * - * A read and its save are two requests, and a fresh lookup on the save can disagree with the one - * that served the read. The Universal Editor Service fetches and posts back to - * `editable.connection.uri.toString()` verbatim, so a param on that uri is what links them. - */ -export const SOURCE_STAMP_PARAM = 'ab-src'; - -const SOURCE_BUS_STAMP = 'sb'; -const LEGACY_STAMP = 'da'; -const NEW_DOCUMENT = 'new'; - -/** - * Stamps a read with the store it came from, and whether it found a document there. - * - * The stamp says nothing about which version was read. One page load produces many saves: the - * editor keeps the connection uri it was served and posts back to it for every edit, so a - * precondition pinned to a version would land the first save and refuse the rest. Nothing on the - * write path can refresh it either, because the source bus sets no etag on a write response. - * - * @param {{kind: string}} source the resolved content source - * @param {boolean} [found] whether the read found a document - * @returns {string} the stamp - */ -export function formatSourceStamp(source, found = false) { - if (source.kind !== SOURCE_BUS) return LEGACY_STAMP; - return found ? SOURCE_BUS_STAMP : `${SOURCE_BUS_STAMP}.${NEW_DOCUMENT}`; -} - -/** - * Reads a stamp back into the store it names and the precondition a write to it carries. - * - * Each precondition holds for every save in a session, not only the first. `If-Match: *` asks the - * source bus that the document exist, which it does once created; a read that found nothing - * carries no precondition, so the save creates the document and the next one overwrites it. - * - * The stamp arrives on a client-supplied url, so a value that is not one of the three forms makes - * the whole stamp unusable rather than being passed to a store. - * - * @param {string} [value] the raw param value - * @returns {{kind: string, condition?: Object}|undefined} undefined when there is no stamp to - * trust, which leaves the fresh lookup to decide on its own - */ -export function parseSourceStamp(value) { - if (value === LEGACY_STAMP) return { kind: LEGACY, condition: undefined }; - if (value === SOURCE_BUS_STAMP) return { kind: SOURCE_BUS, condition: { 'If-Match': '*' } }; - if (value === `${SOURCE_BUS_STAMP}.${NEW_DOCUMENT}`) { - return { kind: SOURCE_BUS, condition: undefined }; - } - return undefined; -} diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index 97c10546..887b6974 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -32,7 +32,7 @@ const recorder = () => { const fetched = []; const env = { DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', + AEM_API: 'https://api.aem.live', daadmin: { fetch: async (input) => { fetched.push(input instanceof Request ? input.url : input.href); @@ -87,7 +87,7 @@ describe('daSourceHead', () => { describe('daSourceGet', () => { const env = { DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', + AEM_API: 'https://api.aem.live', daadmin: { fetch: async () => new Response('stored', { status: 200 }) }, }; diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 34990379..2545f0b9 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -35,7 +35,7 @@ const build = async (overrides = {}) => { // 'headHtml' in overrides rather than a destructured default, so passing // `{ headHtml: undefined }` really does simulate a missing head.html const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; - const seen = { bus: [], legacy: [], stamps: [] }; + const seen = { bus: [], legacy: [], ue: 0 }; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); seen.bus.push({ url: request.url, method: request.method, headers: request.headers }); @@ -43,7 +43,7 @@ const build = async (overrides = {}) => { }; const env = { DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', + AEM_API: 'https://api.aem.live', daadmin: { fetch: async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); @@ -68,7 +68,7 @@ const build = async (overrides = {}) => { serializeHtml: (tree) => `${tree.bodyHtml}`, }, '../../src/ue/ue.js': { - applyUEInstrumentation: async (tree, daCtx, aemCtx, stamp) => { seen.stamps.push(stamp); }, + applyUEInstrumentation: async () => { seen.ue += 1; }, }, '../../src/storage/config.js': { getSiteConfig: async () => { throw new Error('no config'); }, @@ -209,116 +209,62 @@ describe('reading with the content source resolved', () => { }); }); - describe('the stamp a read leaves for its write', () => { - it('names the source bus when the read found a document', async () => { + describe('UE instrumentation', () => { + it('is applied on a UE host', async () => { const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.deepStrictEqual(seen.stamps, ['sb']); + assert.strictEqual(seen.ue, 1); }); - it('says the source-bus document is new when the read found nothing', async () => { - const { daSourceGet, env, seen } = await build({ - source: BUS_SOURCE, - bus: () => new Response('', { status: 404 }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.deepStrictEqual(seen.stamps, ['sb.new']); - }); - - // one page load produces many saves against the same stamp, so pinning the version would - // land the first and refuse the rest - it('carries no version, so it holds for every save in the session', async () => { - const { daSourceGet, env, seen } = await build({ - source: BUS_SOURCE, - bus: () => new Response('x', { status: 200, headers: { etag: '"deadbeefcafe"' } }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.deepStrictEqual(seen.stamps, ['sb']); - }); - - it('names da-admin for a legacy read', async () => { - const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + it('is not applied on a preview host', async () => { + const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const req = authedReq('https://main--site--org.preview.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.deepStrictEqual(seen.stamps, ['da']); + assert.strictEqual(seen.ue, 0); }); - // every other test here mocks applyUEInstrumentation, which leaves the one line that carries - // the stamp from daSourceGet into the emitted meta tag unguarded. These two run the real - // src/ue/ue.js and assert on the served body instead. - describe('as it reaches the served page', () => { - const servedBy = async (source) => { - const seen = { bus: [], legacy: [] }; - globalThis.fetch = async (input, init) => { - const request = input instanceof Request ? input : new Request(input, init); - if (request.url.startsWith('https://api.aem.live/')) { - seen.bus.push(request.url); - return new Response('

x

', { status: 200, headers: { etag: '"busetag"' } }); - } - // component-definition and friends, fetched by the real instrumentation. A missing one - // is what a site without them answers, and getUEConfig degrades to undefined. - return new Response('not found', { status: 404 }); - }; - const env = { - DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', - UE_HOST: 'ue.da.live', - daadmin: { - fetch: async (input, init) => { - const request = input instanceof Request ? input : new Request(input, init); - seen.legacy.push(request.url); - return new Response('

x

', { status: 200 }); - }, - }, - }; - const { daSourceGet } = await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { - default: async () => source, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', - }, - '../../src/utils/aemCtx.js': { - getAemCtx: () => ({ ueHostname: 'ue.da.live', previewUrl: 'https://p.example' }), - getAEMHtml: async () => '', - }, - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - return res.text(); + // the connection uri names the page and nothing else; the store a read came from is not + // carried on it, so a save resolves the store for itself + it('leaves no marker on the connection uri', async () => { + const seenReq = []; + globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + seenReq.push(request.url); + if (request.url.startsWith('https://api.aem.live/')) { + return new Response('

x

', { status: 200 }); + } + return new Response('not found', { status: 404 }); }; - - it('lands on the connection uri for a source-bus read', async () => { - const html = await servedBy(BUS_SOURCE); - - assert.match(html, /urn:adobe:aue:system:ab" content="da:[^"]*\?ab-src=sb"/); - }); - - it('lands on the connection uri for a legacy read', async () => { - const html = await servedBy(LEGACY_SOURCE); - - assert.match(html, /urn:adobe:aue:system:ab" content="da:[^"]*\?ab-src=da"/); + const env = { + DA_ADMIN: 'https://admin.da.live', + AEM_API: 'https://api.aem.live', + UE_HOST: 'ue.da.live', + daadmin: { fetch: async () => new Response('', { status: 200 }) }, + }; + const { daSourceGet } = await esmock('../../src/routes/da-admin.js', { + '../../src/storage/content-source.js': { + default: async () => BUS_SOURCE, + SOURCE_BUS: 'sourcebus', + LEGACY: 'legacy', + UNKNOWN: 'unknown', + }, + '../../src/utils/aemCtx.js': { + getAemCtx: () => ({ ueHostname: 'ue.da.live', previewUrl: 'https://p.example' }), + getAEMHtml: async () => '', + }, }); - }); - - it('is not applied outside UE, where nothing posts back', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); - const req = authedReq('https://main--site--org.preview.da.live/folder/content'); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + const html = await (await daSourceGet({ req, env, daCtx: getDaCtx(req) })).text(); - assert.deepStrictEqual(seen.stamps, []); + assert.match(html, /urn:adobe:aue:system:ab" content="da:[^"?]*"/); + assert.ok(!html.includes('ab-src'), 'no marker on the connection uri'); + assert.ok(!html.includes('content-store'), 'no marker on the connection uri'); }); }); diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 84902b37..d3f1c22e 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -17,8 +17,9 @@ import { getDaCtx } from '../../src/utils/daCtx.js'; const LEGACY_SOURCE = { kind: 'legacy' }; const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; -const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the config service answered 503' }; +const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the API answered 503' }; +const AT = 'https://main--site--org.ue.da.live/folder/content'; const DOC = '

the author typed this

'; /** The shape the Universal Editor Service posts: a `data` blob in a multipart form. */ @@ -29,7 +30,7 @@ const uePost = (url, html = DOC) => { }; const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { - const seen = { bus: [], legacy: [] }; + const seen = { bus: [], legacy: [], lookups: 0 }; const capture = async (request) => { const clone = request.clone(); return { @@ -47,7 +48,7 @@ const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { }; const env = { DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', + AEM_API: 'https://api.aem.live', daadmin: { fetch: async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); @@ -58,7 +59,10 @@ const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { }; const mod = await esmock('../../src/routes/da-admin.js', { '../../src/storage/content-source.js': { - default: async () => source, + default: async () => { + seen.lookups += 1; + return source; + }, SOURCE_BUS: 'sourcebus', LEGACY: 'legacy', UNKNOWN: 'unknown', @@ -67,357 +71,134 @@ const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { return { daSourcePost: mod.daSourcePost, env, seen }; }; -const post = async (opts, url) => { +const post = async (opts, url = AT) => { const { daSourcePost, env, seen } = await build(opts); const req = uePost(url); const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); return { res, seen }; }; -const AT = 'https://main--site--org.ue.da.live/folder/content'; - afterEach(() => { delete globalThis.fetch; }); -describe('a UE session, which saves many times against one page load', () => { - // The Universal Editor Service runs load() then store() for each mutating operation, against - // the connection uri the editor read once at page load - // (universal-editor-service-plugin-da/src/index.ts: add 104/135, copy 150/184, move 242/270, - // patch 301/344, remove 474/492, update 540/555). Every one returns updates[] for in-place DOM - // patching, so the iframe never reloads and the stamp is never refreshed. A precondition that - // pins a version therefore lands the first save and is refused 412 for the rest of the session. - const store = (present) => { - let etag = present ? '"v1"' : undefined; - let body = present ? 'the original' : undefined; - return { - answer: (request) => { - const ifMatch = request.headers.get('If-Match'); - const ifNone = request.headers.get('If-None-Match'); - if (request.method !== 'POST') { - return etag === undefined - ? new Response('', { status: 404 }) - : new Response(body, { status: 200, headers: { etag } }); - } - if (ifNone === '*' && etag !== undefined) return new Response('', { status: 412 }); - if (ifMatch === '*' && etag === undefined) return new Response('', { status: 412 }); - if (ifMatch && ifMatch !== '*' && ifMatch !== etag) return new Response('', { status: 412 }); - body = 'written'; - etag = `"v${Number(etag?.replace(/\D/g, '') ?? 0) + 1}"`; - return new Response('', { status: 201 }); - }, - get body() { return body; }, - }; - }; - - const session = async (present) => { - const st = store(present); - globalThis.fetch = async (input, init) => { - const request = input instanceof Request ? input : new Request(input, init); - return st.answer(request); - }; - const env = { - DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', - daadmin: { fetch: async () => assert.fail('a source-bus site must not touch da-admin') }, - }; - let served; - const mod = await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { - default: async () => BUS_SOURCE, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', - }, - '../../src/utils/aemCtx.js': { - getAemCtx: () => ({}), - getAEMHtml: async () => '', - }, - '../../src/render/compose.js': { - composeHtml: async () => ({}), - serializeHtml: () => 'composed', - }, - '../../src/ue/ue.js': { - applyUEInstrumentation: async (tree, daCtx, aemCtx, stamp) => { served = stamp; }, - }, - '../../src/storage/config.js': { - getSiteConfig: async () => { throw new Error('no config'); }, - }, - }); - - // the editor loads the page once and keeps whatever stamp it was served - const read = new Request(AT, { headers: { Authorization: 'Bearer t' } }); - await mod.daSourceGet({ req: read, env, daCtx: getDaCtx(read) }); - - const at = `${AT}?ab-src=${served}`; - const statuses = []; - for (let i = 0; i < 3; i += 1) { - const request = uePost(at, `

edit ${i + 1}

`); - // eslint-disable-next-line no-await-in-loop - const res = await mod.daSourcePost({ req: request, env, daCtx: getDaCtx(request) }); - statuses.push(res.status); - } - return { statuses, stored: st.body, stamp: served }; - }; - - it('lands all three saves on a page that already existed', async () => { - const { statuses, stamp } = await session(true); - - assert.deepStrictEqual(statuses, [201, 201, 201], `with the stamp the read served: ${stamp}`); - }); - - it('lands all three saves on a page it created', async () => { - const { statuses, stamp } = await session(false); - - assert.deepStrictEqual(statuses, [201, 201, 201], `with the stamp the read served: ${stamp}`); - }); - - it('keeps the last edit, not the first', async () => { - const { stored } = await session(true); - - assert.strictEqual(stored, 'written'); - }); -}); - -describe('writing with the content source resolved', () => { - describe('a stamped source-bus save', () => { - it('goes to the source bus', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); +describe('writing to the store that holds the site', () => { + describe('a source-bus site', () => { + it('writes to the base the config named', async () => { + const { seen } = await post({ source: BUS_SOURCE }); assert.strictEqual(seen.legacy.length, 0); assert.strictEqual(seen.bus.length, 1); assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); }); - it('asks that the document still exist', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); - - assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); - }); - - it('pins no version, so the next save in the session is not refused', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); - - assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); - assert.strictEqual(seen.bus[0].headers.get('If-None-Match'), null); - }); - it('sends the document as the raw body the source bus parses', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); + const { seen } = await post({ source: BUS_SOURCE }); assert.strictEqual(seen.bus[0].body, DOC); assert.strictEqual(seen.bus[0].contentType, 'text/html'); }); // helix-api-service parses no form data; the envelope would be stored as the document text + // and answered 201 it('never wraps the body in a multipart envelope', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb`); + const { seen } = await post({ source: BUS_SOURCE }); assert.ok(!seen.bus[0].body.includes('Content-Disposition'), seen.bus[0].body); assert.ok(!/boundary/i.test(seen.bus[0].contentType ?? ''), seen.bus[0].contentType); }); - it('passes the store answer back, so a 412 reaches the editor', async () => { - const { res } = await post({ source: BUS_SOURCE, status: 412 }, `${AT}?ab-src=sb`); + it('authorizes with the caller token', async () => { + const { seen } = await post({ source: BUS_SOURCE }); - assert.strictEqual(res.status, 412); + assert.strictEqual(seen.bus[0].headers.get('Authorization'), 'Bearer t'); }); - }); - describe('a save of a page the read did not find', () => { - // If-None-Match: * would create on the first save and refuse every one after it - it('carries no precondition, so it creates and then overwrites', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.new`); + it('passes the store answer back', async () => { + const { res } = await post({ source: BUS_SOURCE, status: 412 }); - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.bus[0].headers.get('If-None-Match'), null); - assert.strictEqual(seen.bus[0].headers.get('If-Match'), null); + assert.strictEqual(res.status, 412); }); }); - describe('a stamped legacy save', () => { - it('goes to da-admin as a data form part', async () => { - const { seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=da`); + describe('a legacy site', () => { + it('writes to da-admin as a data form part', async () => { + const { seen } = await post({ source: LEGACY_SOURCE }); assert.strictEqual(seen.bus.length, 0); assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); assert.ok(seen.legacy[0].body.includes('name="data"'), seen.legacy[0].body); assert.ok(seen.legacy[0].body.includes('the author typed this'), seen.legacy[0].body); }); - - // da-admin sets no etag on a read, so there is nothing to condition on - it('carries no precondition', async () => { - const { seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=da`); - - assert.strictEqual(seen.legacy[0].headers.get('If-Match'), null); - assert.strictEqual(seen.legacy[0].headers.get('If-None-Match'), null); - }); }); - describe('when the store moved between the read and the save', () => { - // the destructive case. The author read da-admin's copy, which for a migrated site is its - // stale pre-migration content, and the site is now served from the source bus. Writing it - // there would overwrite the live page with content the author never saw. - it('refuses a legacy-stamped save to a site now on the source bus', async () => { - const { res, seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=da`); - - assert.strictEqual(res.status, 409); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); - }); - - it('refuses a source-bus-stamped save to a site now on da-admin', async () => { - const { res, seen } = await post({ source: LEGACY_SOURCE }, `${AT}?ab-src=sb`); - - assert.strictEqual(res.status, 409); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); - }); - - it('says so in plain text, since nothing renders a refused write', async () => { - const { res } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=da`); - - assert.match(res.headers.get('Content-Type'), /^text\/plain/); - assert.ok((await res.text()).length > 0); - }); - }); - - describe('a save with no stamp', () => { - it('goes to da-admin unconditionally on a legacy site, as it did before', async () => { - const { res, seen } = await post({ source: LEGACY_SOURCE }, AT); - - assert.strictEqual(res.status, 201); - assert.strictEqual(seen.legacy.length, 1); - assert.strictEqual(seen.legacy[0].headers.get('If-Match'), null); - }); - - // an old page, or a stamp the editor dropped. There is no provenance, so the save may - // overwrite a page that exists but may not invent a new one. - it('may overwrite but not create on a source-bus site', async () => { - const { seen } = await post({ source: BUS_SOURCE }, AT); - - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); - }); - }); + describe('no write carries a precondition', () => { + // only the source bus sets an etag on a read, and nothing round-trips it into the save: the + // editor keeps the connection uri it was served at page load and posts back to it for every + // edit, so a version pin would land the first save and refuse the rest with 412 + [['a source-bus', BUS_SOURCE, 'bus'], ['a legacy', LEGACY_SOURCE, 'legacy']].forEach( + ([what, source, where]) => { + it(`${what} write sends no If-Match or If-None-Match`, async () => { + const { seen } = await post({ source }); + + assert.strictEqual(seen[where][0].headers.get('If-Match'), null); + assert.strictEqual(seen[where][0].headers.get('If-None-Match'), null); + }); + }, + ); - describe('a save carrying a stamp that cannot be trusted', () => { - [ - ['an unknown store', 'gcs.abc'], - ['a version pin, which this no longer emits', 'sb.9e8311043aab12b1'], - ['a value with a quote in it', 'sb.a"b'], - ['a header injection attempt', 'sb.abc%0d%0aX-Evil:%201'], - ['an empty value', ''], - ].forEach(([what, value]) => { - it(`falls back to no stamp: ${what}`, async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=${value}`); - - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.bus[0].headers.get('If-Match'), '*'); - }); - }); + it('so a UE session can save the same page many times', async () => { + const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); + const statuses = []; - it('never lets a stamp put a raw newline in a header', async () => { - const { seen } = await post({ source: BUS_SOURCE }, `${AT}?ab-src=sb.a%0db`); + for (let i = 0; i < 4; i += 1) { + const req = uePost(AT, `

edit ${i + 1}

`); + // eslint-disable-next-line no-await-in-loop + const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + statuses.push(res.status); + } - assert.ok(!/[\r\n]/.test(seen.bus[0].headers.get('If-Match') ?? '')); + assert.deepStrictEqual(statuses, [201, 201, 201, 201]); + assert.strictEqual(seen.bus.length, 4); }); }); - describe('how fresh the lookup on a write is', () => { - // reads reuse an answer for a few seconds so one page's images do not each pay for it. A - // write must not: a stale answer would let the 409 miss a site that moved stores in that - // window, which is the one case that overwrites a live page. - const optsSeenBy = async (drive) => { - const opts = []; - globalThis.fetch = async () => new Response('', { status: 201 }); - const env = { - DA_ADMIN: 'https://admin.da.live', - HLX_ADMIN: 'https://admin.hlx.page', - daadmin: { fetch: async () => new Response('', { status: 201 }) }, - }; - const mod = await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { - default: async (e, ctx, o) => { - opts.push(o); - return BUS_SOURCE; - }, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', - }, - '../../src/utils/aemCtx.js': { - getAemCtx: () => ({}), - getAEMHtml: async () => '', - }, - '../../src/render/compose.js': { - composeHtml: async () => ({}), - serializeHtml: () => 'c', - }, - '../../src/ue/ue.js': { applyUEInstrumentation: async () => {} }, - '../../src/storage/config.js': { - getSiteConfig: async () => { throw new Error('none'); }, - }, - }); - await drive(mod, env); - return opts; - }; + describe('the store lookup on a write', () => { + // nothing is held between requests, so a write and the read before it resolve independently + it('happens once per write', async () => { + const { seen } = await post({ source: BUS_SOURCE }); - it('asks for a fresh answer on a write', async () => { - const opts = await optsSeenBy(async (mod, env) => { - const request = uePost(`${AT}?ab-src=sb`); - await mod.daSourcePost({ req: request, env, daCtx: getDaCtx(request) }); - }); - - assert.strictEqual(opts.length, 1); - assert.notStrictEqual(opts[0]?.reuse, true); + assert.strictEqual(seen.lookups, 1); }); - it('lets a read reuse one', async () => { - const opts = await optsSeenBy(async (mod, env) => { - const request = new Request(AT, { headers: { Authorization: 'Bearer t' } }); - await mod.daSourceGet({ req: request, env, daCtx: getDaCtx(request) }); - }); - - assert.strictEqual(opts[0]?.reuse, true); - }); + it('happens before anything is sent to a store', async () => { + const { seen } = await post({ source: UNKNOWN_SOURCE }); - it('lets a HEAD reuse one', async () => { - const opts = await optsSeenBy(async (mod, env) => { - const request = new Request(AT, { headers: { Authorization: 'Bearer t' } }); - await mod.daSourceHead({ env, daCtx: getDaCtx(request) }); - }); - - assert.strictEqual(opts[0]?.reuse, true); - }); - - it('lets a non-html read reuse one', async () => { - const opts = await optsSeenBy(async (mod, env) => { - const url = 'https://main--site--org.ue.da.live/photo.png'; - const request = new Request(url, { headers: { Authorization: 'Bearer t' } }); - await mod.daSourceGet({ req: request, env, daCtx: getDaCtx(request) }); - }); - - assert.strictEqual(opts[0]?.reuse, true); + assert.strictEqual(seen.lookups, 1); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); }); }); describe('when the content source could not be resolved', () => { it('refuses with 503 and touches neither store', async () => { - const { res, seen } = await post({ source: UNKNOWN_SOURCE }, `${AT}?ab-src=sb`); + const { res, seen } = await post({ source: UNKNOWN_SOURCE }); assert.strictEqual(res.status, 503); assert.strictEqual(seen.bus.length + seen.legacy.length, 0); }); it('asks the caller to retry', async () => { - const { res } = await post({ source: UNKNOWN_SOURCE }, AT); + const { res } = await post({ source: UNKNOWN_SOURCE }); assert.ok(Number(res.headers.get('Retry-After')) > 0); }); + // nothing renders a POST body, and UES embeds it verbatim in its problem+json error string it('says so in plain text', async () => { - const { res } = await post({ source: UNKNOWN_SOURCE }, AT); + const { res } = await post({ source: UNKNOWN_SOURCE }); assert.match(res.headers.get('Content-Type'), /^text\/plain/); + assert.ok((await res.text()).length > 0); }); }); @@ -425,7 +206,7 @@ describe('writing with the content source resolved', () => { it('strips the UE data attributes before the store sees them', async () => { const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); const req = uePost( - `${AT}?ab-src=sb`, + AT, '

text

', ); @@ -441,6 +222,7 @@ describe('writing with the content source resolved', () => { const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); assert.strictEqual(res.status, 415); + assert.strictEqual(seen.lookups, 0); assert.strictEqual(seen.bus.length + seen.legacy.length, 0); }); }); @@ -449,7 +231,7 @@ describe('writing with the content source resolved', () => { it('keeps the case the source bus stores it under', async () => { const { seen } = await post( { source: BUS_SOURCE }, - 'https://main--site--org.ue.da.live/Folder/Content?ab-src=sb', + 'https://main--site--org.ue.da.live/Folder/Content', ); assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Folder/content.html'); @@ -458,7 +240,7 @@ describe('writing with the content source resolved', () => { it('lowercases the whole path for da-admin', async () => { const { seen } = await post( { source: LEGACY_SOURCE }, - 'https://main--site--org.ue.da.live/Folder/Content?ab-src=da', + 'https://main--site--org.ue.da.live/Folder/Content', ); assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index 44e42dbb..787b2db9 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -15,7 +15,7 @@ import assert from 'assert'; const { default: resolveContentSource } = await import('../../src/storage/content-source.js'); -const env = { HLX_ADMIN: 'https://admin.hlx.page' }; +const env = { AEM_API: 'https://api.aem.live' }; const daCtx = (over = {}) => ({ org: 'org', site: 'site', ref: 'main', authToken: 'Bearer t', ...over, @@ -44,16 +44,23 @@ describe('resolveContentSource', () => { }); describe('the request it makes', () => { - it('asks the sidekick config for org, site and ref', async () => { + it('asks the AEM API for the site', async () => { stubFetch(legacyBody); - await resolveContentSource(env, daCtx({ ref: 'branch' })); + await resolveContentSource(env, daCtx()); assert.strictEqual(calls.length, 1); - assert.strictEqual( - calls[0].url, - 'https://admin.hlx.page/sidekick/org/site/branch/config.json', - ); + assert.strictEqual(calls[0].url, 'https://api.aem.live/org/sites/site/sidekick'); + }); + + // both stores read the same config service and the source is per site, so the branch does + // not change the answer + it('does not vary by ref', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx({ ref: 'branch' })); + + assert.strictEqual(calls[0].url, 'https://api.aem.live/org/sites/site/sidekick'); }); it('passes the author token on, so a private site resolves', async () => { @@ -129,7 +136,7 @@ describe('resolveContentSource', () => { assert.strictEqual(source.kind, 'unknown'); }); - it('answers unknown for a host that only starts like api.aem.live', async () => { + it('answers unknown for a host that only starts like the API', async () => { stubFetch(() => sidekick('https://api.aem.live.evil.example/org/sites/site/source')); const source = await resolveContentSource(env, daCtx()); @@ -222,121 +229,49 @@ describe('resolveContentSource', () => { }); }); - describe('reusing an answer within a page load', () => { - // one previewed page is many worker requests: the document, then one per relative image src, - // each of which would look the store up again. Measured live on 2026-08-03: the sidekick - // config is `cache-control: no-store` and costs ~460ms, so 8 identical lookups spend 3.8s of - // origin time and add that to every image. - // - // What is held is per isolate and shared across requests, so each test here uses its own site. - it('asks once for a burst of reads of the same site', async () => { - stubFetch(legacyBody); - const ctx = daCtx({ site: 'burst' }); - - await resolveContentSource(env, ctx, { reuse: true }); - await resolveContentSource(env, ctx, { reuse: true }); - await resolveContentSource(env, ctx, { reuse: true }); - - assert.strictEqual(calls.length, 1); - }); - - it('gives the same answer each time', async () => { - stubFetch(() => sidekick('https://api.aem.live/org/sites/same/source')); - const ctx = daCtx({ site: 'same' }); - - const first = await resolveContentSource(env, ctx, { reuse: true }); - const second = await resolveContentSource(env, ctx, { reuse: true }); - - assert.deepStrictEqual(second, first); - }); - - it('asks again for a different site', async () => { + describe('the API host', () => { + // the caller turns unknown into a 503 it can return; a throw here escapes into + // withCorsHeaders, which reads response.headers and throws again on undefined + it('answers unknown rather than throwing when it is not set', async () => { stubFetch(legacyBody); - await resolveContentSource(env, daCtx({ site: 'one' }), { reuse: true }); - await resolveContentSource(env, daCtx({ site: 'two' }), { reuse: true }); + const source = await resolveContentSource({}, daCtx()); - assert.strictEqual(calls.length, 2); + assert.strictEqual(source.kind, 'unknown'); }); - it('asks again for a different ref of the same site', async () => { + it('answers unknown rather than throwing when it is not a url', async () => { stubFetch(legacyBody); - await resolveContentSource(env, daCtx({ site: 'refs' }), { reuse: true }); - await resolveContentSource(env, daCtx({ site: 'refs', ref: 'branch' }), { reuse: true }); + const source = await resolveContentSource({ AEM_API: 'not-a-url' }, daCtx()); - assert.strictEqual(calls.length, 2); + assert.strictEqual(source.kind, 'unknown'); }); - // a write is the one operation a wrong store cannot be walked back from, so it always asks - it('does not reuse an answer unless asked to', async () => { + it('comes from env, so stage can point elsewhere', async () => { stubFetch(legacyBody); - const ctx = daCtx({ site: 'writes' }); - await resolveContentSource(env, ctx, { reuse: true }); - await resolveContentSource(env, ctx); - await resolveContentSource(env, ctx); + await resolveContentSource({ AEM_API: 'https://api.stage.example' }, daCtx()); - assert.strictEqual(calls.length, 3); + assert.strictEqual(calls[0].url, 'https://api.stage.example/org/sites/site/sidekick'); }); - // an outage that stuck would outlast itself - it('never stores an answer it could not give', async () => { - stubFetch(() => new Response('', { status: 503 })); - const ctx = daCtx({ site: 'flaky' }); - - await resolveContentSource(env, ctx, { reuse: true }); - await resolveContentSource(env, ctx, { reuse: true }); - - assert.strictEqual(calls.length, 2); - }); - - it('picks up a recovery on the next read', async () => { - let attempt = 0; - stubFetch(() => { - attempt += 1; - return attempt === 1 - ? new Response('', { status: 503 }) - : sidekick('https://api.aem.live/org/sites/recovering/source'); - }); - const ctx = daCtx({ site: 'recovering' }); - - const down = await resolveContentSource(env, ctx, { reuse: true }); - const up = await resolveContentSource(env, ctx, { reuse: true }); - - assert.strictEqual(down.kind, 'unknown'); - assert.strictEqual(up.kind, 'sourcebus'); - }); - }); - - describe('the admin host', () => { - // the caller turns unknown into a 503 it can return; a throw here escapes into - // withCorsHeaders, which reads response.headers and throws again on undefined - it('answers unknown rather than throwing when it is not set', async () => { + it('tolerates a trailing slash on it', async () => { stubFetch(legacyBody); - const source = await resolveContentSource({}, daCtx()); + await resolveContentSource({ AEM_API: 'https://api.aem.live/' }, daCtx()); - assert.strictEqual(source.kind, 'unknown'); + assert.strictEqual(calls[0].url, 'https://api.aem.live/org/sites/site/sidekick'); }); - it('answers unknown rather than throwing when it is not a url', async () => { - stubFetch(legacyBody); + // the same env value decides where we ask and what counts as the source bus, so pointing at + // stage must not leave the prefix test matching production + it('is also what makes a source url the source bus', async () => { + stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source')); - const source = await resolveContentSource({ HLX_ADMIN: 'not-a-url' }, daCtx()); + const source = await resolveContentSource({ AEM_API: 'https://api.stage.example' }, daCtx()); assert.strictEqual(source.kind, 'unknown'); }); - - it('comes from env, so stage can point elsewhere', async () => { - stubFetch(legacyBody); - - await resolveContentSource({ HLX_ADMIN: 'https://admin.stage.example' }, daCtx()); - - assert.strictEqual( - calls[0].url, - 'https://admin.stage.example/sidekick/org/site/main/config.json', - ); - }); }); }); diff --git a/test/storage/store.test.js b/test/storage/store.test.js index 8fbaa8f8..38ae23ae 100644 --- a/test/storage/store.test.js +++ b/test/storage/store.test.js @@ -162,21 +162,19 @@ describe('getStore', () => { assert.strictEqual(new Headers(l.writeInit('', 'Bearer t').headers).get('Authorization'), 'Bearer t'); }); - it('adds a precondition to the source-bus write when one is given', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); - - const init = store.writeInit('', 'Bearer t', { 'If-Match': '"abc"' }); - - assert.strictEqual(new Headers(init.headers).get('If-Match'), '"abc"'); - }); - - it('sends no precondition when none is given', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); - - const init = store.writeInit('', 'Bearer t'); + // neither store's writes are conditional: only the source bus sets an etag on a read, and a + // marker on the connection uri is minted once per page load while UE saves many times against + // it, so nothing could refresh a version pin + it('sends no precondition to either store', () => { + const b = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + const l = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); - assert.strictEqual(new Headers(init.headers).get('If-Match'), null); - assert.strictEqual(new Headers(init.headers).get('If-None-Match'), null); + [b, l].forEach((store) => { + const headers = new Headers(store.writeInit('', 'Bearer t').headers); + assert.strictEqual(headers.get('If-Match'), null); + assert.strictEqual(headers.get('If-None-Match'), null); + assert.strictEqual(headers.get('If-Unmodified-Since'), null); + }); }); }); }); diff --git a/test/ue/source-stamp-scaffold.test.js b/test/ue/source-stamp-scaffold.test.js deleted file mode 100644 index 48474786..00000000 --- a/test/ue/source-stamp-scaffold.test.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2026 Adobe. All rights reserved. - * This file is licensed to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS - * OF ANY KIND, either express or implied. See the License for the specific language - * governing permissions and limitations under the License. - */ - -/* eslint-env mocha */ -import assert from 'assert'; -import { getUEHtmlHeadEntries } from '../../src/ue/scaffold.js'; -import { getAemCtx } from '../../src/utils/aemCtx.js'; -import { UNAUTHORIZED_HTML_MESSAGE } from '../../src/utils/constants.js'; - -const env = { UE_HOST: 'test-ue-host', UE_SERVICE: 'test-ue-service' }; - -const connectionContent = (daCtx, stamp) => { - const entries = getUEHtmlHeadEntries(daCtx, getAemCtx(env, daCtx), stamp); - return entries.find((e) => e.properties?.name === 'urn:adobe:aue:system:ab').properties.content; -}; - -const hosted = { - org: 'org', site: 'site', ref: 'ref', path: '/some-path', aemPathname: '/some-path', -}; -const local = { ...hosted, isLocal: true, orgSiteInPath: true }; - -describe('the stamp on the UE connection uri', () => { - it('is absent when the read left none', () => { - assert.strictEqual( - connectionContent(hosted, undefined), - 'da:https://ref--site--org.test-ue-host/some-path', - ); - }); - - it('is carried as a query param the Universal Editor Service posts back', () => { - assert.strictEqual( - connectionContent(hosted, 'sb.abc123'), - 'da:https://ref--site--org.test-ue-host/some-path?ab-src=sb.abc123', - ); - }); - - it('is carried on the localhost form too', () => { - assert.strictEqual( - connectionContent(local, 'da'), - 'da:https://test-ue-host/org/site/some-path?ab-src=da', - ); - }); - - it('keeps the uri parseable, so new URL() round-trips it', () => { - const content = connectionContent(hosted, 'sb.abc123'); - const uri = content.replace(/^da:/, ''); - - assert.strictEqual(new URL(uri).toString(), uri); - assert.strictEqual(new URL(uri).searchParams.get('ab-src'), 'sb.abc123'); - }); - - it('leaves the path and host untouched, so gimme_cookie still resolves', () => { - const uri = new URL(connectionContent(hosted, 'sb.abc123').replace(/^da:/, '')); - - assert.strictEqual(uri.pathname, '/some-path'); - assert.strictEqual(uri.hostname, 'ref--site--org.test-ue-host'); - assert.strictEqual(new URL('/gimme_cookie', uri).toString(), 'https://ref--site--org.test-ue-host/gimme_cookie'); - }); - - it('is not added to the 401 sentinel, which the authorbus extension matches exactly', () => { - // the shipped extension compares the endpoint to the literal '401' and 'da://401' and on a - // match refetches /gimme_cookie and refreshes the page; a stamp would stop that firing - assert.ok(UNAUTHORIZED_HTML_MESSAGE.includes('content="da:401"')); - assert.ok(!UNAUTHORIZED_HTML_MESSAGE.includes('ab-src')); - }); -}); diff --git a/test/utils/source-stamp.test.js b/test/utils/source-stamp.test.js deleted file mode 100644 index 7fc236b5..00000000 --- a/test/utils/source-stamp.test.js +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2026 Adobe. All rights reserved. - * This file is licensed to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS - * OF ANY KIND, either express or implied. See the License for the specific language - * governing permissions and limitations under the License. - */ - -/* eslint-env mocha */ -import assert from 'assert'; -import { LEGACY, SOURCE_BUS } from '../../src/storage/content-source.js'; - -const { - SOURCE_STAMP_PARAM, formatSourceStamp, parseSourceStamp, -} = await import('../../src/utils/source-stamp.js'); - -const bus = { kind: SOURCE_BUS, base: 'https://api.aem.live/org/sites/site/source' }; -const legacy = { kind: LEGACY }; - -describe('formatSourceStamp', () => { - it('names the param the connection uri carries', () => { - assert.strictEqual(SOURCE_STAMP_PARAM, 'ab-src'); - }); - - it('stamps a source-bus read that found a document', () => { - assert.strictEqual(formatSourceStamp(bus, true), 'sb'); - }); - - it('stamps a source-bus read that found nothing', () => { - assert.strictEqual(formatSourceStamp(bus, false), 'sb.new'); - }); - - // one page load produces many saves against the same stamp, so a version in it would land the - // first save and refuse the rest - it('carries no version, whatever etag the read returned', () => { - assert.doesNotMatch(formatSourceStamp(bus, true), /[0-9a-f]{8}/); - }); - - it('needs no url encoding', () => { - ['sb', 'sb.new', 'da'].forEach((v) => assert.strictEqual(encodeURIComponent(v), v)); - }); - - it('stamps a legacy read, which has no version to carry either', () => { - assert.strictEqual(formatSourceStamp(legacy, false), 'da'); - }); - - it('stamps a legacy read the same whether or not the document was found', () => { - assert.strictEqual(formatSourceStamp(legacy, true), 'da'); - }); -}); - -describe('parseSourceStamp', () => { - describe('a source-bus stamp', () => { - it('reads the store back', () => { - assert.strictEqual(parseSourceStamp('sb').kind, SOURCE_BUS); - assert.strictEqual(parseSourceStamp('sb.new').kind, SOURCE_BUS); - }); - - it('asks that the document still exist, which holds for every save in a session', () => { - assert.deepStrictEqual(parseSourceStamp('sb').condition, { 'If-Match': '*' }); - }); - - // If-None-Match: * would refuse every save after the one that created the document - it('carries no precondition when the read found nothing, so the save can create it', () => { - assert.strictEqual(parseSourceStamp('sb.new').condition, undefined); - }); - }); - - describe('a legacy stamp', () => { - it('reads the store back', () => { - assert.strictEqual(parseSourceStamp('da').kind, LEGACY); - }); - - // da-admin sets no etag on a source GET or HEAD, only on a POST response, so a read there - // cannot produce a precondition. Verified live on 2026-08-03. - it('carries no precondition, since a legacy read yields no etag', () => { - assert.strictEqual(parseSourceStamp('da').condition, undefined); - }); - }); - - describe('anything else', () => { - [ - ['no stamp at all', null], - ['an empty stamp', ''], - ['an unknown store', 'gcs.abc'], - ['a stamp shaped like a path', 'sb/abc'], - ['a version pin, which this no longer emits', 'sb.9e8311043aab12b1'], - ['a url-unsafe character', 'sb.a"b'], - ['a slash', 'sb.a/b'], - ['a trailing dot', 'sb.'], - ['a header injection attempt', 'sb.abc\r\nX-Evil: 1'], - ['a case variation', 'SB'], - ].forEach(([what, value]) => { - it(`is not trusted: ${what}`, () => { - assert.strictEqual(parseSourceStamp(value), undefined); - }); - }); - }); - - describe('round trip', () => { - [[bus, true], [bus, false], [legacy, true], [legacy, false]].forEach(([source, found]) => { - it(`parses back what a ${source.kind} read stamped, found=${found}`, () => { - const parsed = parseSourceStamp(formatSourceStamp(source, found)); - - assert.ok(parsed, 'a stamp this code emits must parse back'); - assert.strictEqual(parsed.kind, source.kind); - }); - }); - }); -}); diff --git a/wrangler.toml b/wrangler.toml index 73dba068..713c655e 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,18 +2,18 @@ name = "da-ue" main = "src/index.js" compatibility_date = "2023-11-21" -vars = { UE_HOST = "ue.da.live", DA_ADMIN = "https://admin.da.live", HLX_ADMIN = "https://admin.hlx.page" } +vars = { UE_HOST = "ue.da.live", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live" } services = [{ binding = "daadmin", service = "da-admin" }] [dev] port = 4712 [env.dev] -vars = { ENVIRONMENT = "dev", UE_HOST = "localhost:4712", UE_SERVICE = "https://localhost:8000", DA_ADMIN = "https://admin.da.live", HLX_ADMIN = "https://admin.hlx.page" } +vars = { ENVIRONMENT = "dev", UE_HOST = "localhost:4712", UE_SERVICE = "https://localhost:8000", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live" } services = [{ binding = "daadmin", service = "da-admin-local" }] [env.stage] -vars = { ENVIRONMENT = "stage", UE_HOST = "stage-ue.da.live", UE_SERVICE = "https://universal-editor-service-dev.adobe.io", DA_ADMIN = "https://admin.da.live", HLX_ADMIN = "https://admin.hlx.page" } +vars = { ENVIRONMENT = "stage", UE_HOST = "stage-ue.da.live", UE_SERVICE = "https://universal-editor-service-dev.adobe.io", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live" } services = [{ binding = "daadmin", service = "da-admin" }] [env.stage.observability] From 9d75dd7c51344dd04bc06531e07cb2d4210df047 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 10:17:10 +0200 Subject: [PATCH 30/48] test: pin that auth failures answer 401 and an unreachable store answers 503 reporting an expired session as unresolved answers a retryable 503, so the client never re-authenticates and the da:401 recovery the authorbus extension has never fires. and a throw from either store's fetch escapes into withCorsHeaders, which reads response.headers and turns it into an opaque 500 with no CORS. --- test/routes/source-read.test.js | 104 ++++++++++++++++++++++++++++ test/routes/source-write.test.js | 23 ++++++ test/storage/content-source.test.js | 39 ++++++++--- 3 files changed, 158 insertions(+), 8 deletions(-) diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 2545f0b9..be10701e 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -18,6 +18,8 @@ import { getDaCtx } from '../../src/utils/daCtx.js'; const LEGACY_SOURCE = { kind: 'legacy' }; const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the config service answered 503' }; +const DENIED_SOURCE = { kind: 'unauthorized', status: 401 }; +const FORBIDDEN_SOURCE = { kind: 'unauthorized', status: 403 }; const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer t' } }); @@ -178,6 +180,108 @@ describe('reading with the content source resolved', () => { }); }); + describe('when the caller is not allowed to ask which store', () => { + it('answers 401 on a GET, so the client knows to re-authenticate', async () => { + const { daSourceGet, env, seen } = await build({ source: DENIED_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 401); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + // the authorbus extension matches this sentinel exactly and recovers by refetching + // /gimme_cookie and refreshing the page + it('serves the da:401 shell the editor recovers from', async () => { + const { daSourceGet, env } = await build({ source: DENIED_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.match(await res.text(), /content="da:401"/); + }); + + it('does not ask the caller to retry, since retrying cannot help', async () => { + const { daSourceGet, env } = await build({ source: DENIED_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('Retry-After'), null); + }); + + it('passes a 403 through as itself', async () => { + const { daSourceGet, env } = await build({ source: FORBIDDEN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 403); + }); + + it('answers 401 on a non-html GET too', async () => { + const { daSourceGet, env } = await build({ source: DENIED_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 401); + }); + + it('answers 401 on a HEAD with no body', async () => { + const { daSourceHead, env } = await build({ source: DENIED_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 401); + assert.strictEqual(await res.text(), ''); + }); + }); + + describe('when the store cannot be reached at all', () => { + // withCorsHeaders reads response.headers, so a throw escaping a handler is an opaque 500 + // with no CORS headers on it + it('answers 503 rather than throwing on an html read', async () => { + const { daSourceGet, env } = await build({ + source: BUS_SOURCE, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + }); + + it('answers 503 rather than throwing on a non-html read', async () => { + const { daSourceGet, env } = await build({ + source: BUS_SOURCE, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 503); + }); + + it('answers 503 rather than throwing on a HEAD', async () => { + const { daSourceHead, env } = await build({ + source: BUS_SOURCE, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + assert.strictEqual((await daSourceHead({ env, daCtx: getDaCtx(req) })).status, 503); + }); + + it('answers 503 rather than throwing when da-admin is unreachable', async () => { + const { daSourceGet, env } = await build({ + source: LEGACY_SOURCE, + legacy: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 503); + }); + }); + describe('what a store status means', () => { // turning any of these into the blank starter template at HTTP 200 hands the author an // empty document to save over a page that exists diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index d3f1c22e..fddab2de 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -18,6 +18,7 @@ import { getDaCtx } from '../../src/utils/daCtx.js'; const LEGACY_SOURCE = { kind: 'legacy' }; const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the API answered 503' }; +const DENIED_SOURCE = { kind: 'unauthorized', status: 401 }; const AT = 'https://main--site--org.ue.da.live/folder/content'; const DOC = '

the author typed this

'; @@ -202,6 +203,28 @@ describe('writing to the store that holds the site', () => { }); }); + describe('when the caller is not allowed to ask which store', () => { + it('answers 401, not a retryable 503', async () => { + const { res, seen } = await post({ source: DENIED_SOURCE }); + + assert.strictEqual(res.status, 401); + assert.strictEqual(res.headers.get('Retry-After'), null); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + }); + + describe('when the store cannot be reached at all', () => { + it('answers 503 rather than throwing', async () => { + const { daSourcePost, env } = await build({ source: BUS_SOURCE }); + globalThis.fetch = async () => { throw new TypeError('fetch failed'); }; + const req = uePost(AT); + + const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + }); + }); + describe('what is written', () => { it('strips the UE data attributes before the store sees them', async () => { const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index 787b2db9..fdd48a68 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -183,14 +183,6 @@ describe('resolveContentSource', () => { assert.strictEqual(source.kind, 'unknown'); }); - it('answers unknown on a 401', async () => { - stubFetch(() => new Response('', { status: 401 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - it('answers unknown when the body is not json', async () => { stubFetch(() => new Response('gateway', { status: 200 })); @@ -218,6 +210,37 @@ describe('resolveContentSource', () => { }); }); + describe('when the caller is not allowed to ask', () => { + // "we do not know" is retryable and "you are not authenticated" is not. Reporting an expired + // session as unknown turns into a 503 that says retry, so the client never re-authenticates + // and the da:401 recovery the authorbus extension has never fires. + [401, 403].forEach((status) => { + it(`answers unauthorized on a ${status}, not unknown`, async () => { + stubFetch(() => new Response('', { status })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unauthorized'); + }); + + it(`carries the ${status} through, so the caller answers the same`, async () => { + stubFetch(() => new Response('', { status })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.status, status); + }); + }); + + it('still answers unknown for a 5xx, which is retryable', async () => { + stubFetch(() => new Response('', { status: 502 })); + + const source = await resolveContentSource(env, daCtx()); + + assert.strictEqual(source.kind, 'unknown'); + }); + }); + describe('when there is no site to ask about', () => { it('answers unknown without making a request', async () => { stubFetch(legacyBody); From 5eed17b4a26bfcb07cd8603431054902c69d38c0 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 10:18:17 +0200 Subject: [PATCH 31/48] fix: answer 401 on an auth failure, and 503 when a store will not answer a 401 or 403 from the lookup means the caller is not authorized, which is a definite answer. reporting it as unresolved answered a retryable 503, so the client kept retrying a session that cannot recover and the da:401 shell the authorbus extension recovers from was never served. and a throw from either store's fetch escaped into withCorsHeaders, which reads response.headers, making it an opaque 500 with no CORS. --- src/routes/da-admin.js | 43 ++++++++++++++++++++++++++++---- src/storage/content-source.js | 13 ++++++++-- src/utils/constants.js | 4 +++ test/routes/source-write.test.js | 4 ++- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index bc346cb4..fd4fb1ff 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -27,12 +27,14 @@ import { import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, + SOURCE_UNREACHABLE_HTML_MESSAGE, + SOURCE_UNREACHABLE_MESSAGE, SOURCE_UNRESOLVED_HTML_MESSAGE, SOURCE_UNRESOLVED_MESSAGE, UNAUTHORIZED_HTML_MESSAGE, } from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; -import resolveContentSource, { UNKNOWN } from '../storage/content-source.js'; +import resolveContentSource, { UNAUTHORIZED, UNKNOWN } from '../storage/content-source.js'; import getStore from '../storage/store.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; @@ -84,6 +86,21 @@ async function getPageTemplate(env, daCtx, aemCtx) { return DEFAULT_HTML_TEMPLATE; } +/** + * Sends a request to a store and answers 503 when it could not be reached at all. + * + * A throw escaping a handler reaches withCorsHeaders, which reads `response.headers` and throws + * again, so the caller gets an opaque 500 with no CORS headers on it. + */ +async function reachStore(store, input, init) { + try { + return await store.fetch(input, init); + } catch (e) { + console.warn(`503 ${store.url}, the store could not be reached: ${e.name}: ${e.message}`); + return undefined; + } +} + export async function daSourceGet({ req, env, daCtx }) { const { ext, authToken } = daCtx; @@ -106,13 +123,17 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing const source = await resolveContentSource(env, daCtx); + if (source.kind === UNAUTHORIZED) { + return daResp({ body: UNAUTHORIZED_HTML_MESSAGE, status: source.status, contentType: 'text/html' }); + } if (source.kind === UNKNOWN) { console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); } const store = getStore(env, daCtx, source); console.log(`-> ${store.url.toString()}`); - const response = await store.fetch(store.url, { method: 'GET', headers }); + const response = await reachStore(store, store.url, { method: 'GET', headers }); + if (!response) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } @@ -131,6 +152,9 @@ export async function daSourceGet({ req, env, daCtx }) { } return get404(BRANCH_NOT_FOUND_HTML_MESSAGE); } + if (source.kind === UNAUTHORIZED) { + return daResp({ body: UNAUTHORIZED_HTML_MESSAGE, status: source.status, contentType: 'text/html' }); + } if (source.kind === UNKNOWN) { console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); @@ -145,7 +169,8 @@ export async function daSourceGet({ req, env, daCtx }) { headers, }); console.log(`-> ${store.url.toString()}`); - const sourceResp = await store.fetch(req); + const sourceResp = await reachStore(store, req); + if (!sourceResp) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); console.log(`<- ${store.url.toString()}. ${sourceResp.status} ${sourceResp.statusText}`, { status: sourceResp.status, statusText: sourceResp.statusText }); // only a 404 means "this document is not here". Composing the starter template over anything @@ -197,6 +222,9 @@ export async function daSourceHead({ env, daCtx }) { headers.set('Authorization', authToken); const source = await resolveContentSource(env, daCtx); + if (source.kind === UNAUTHORIZED) { + return new Response(null, { status: source.status }); + } if (source.kind === UNKNOWN) { console.warn(`503 HEAD ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return head503(); @@ -204,7 +232,8 @@ export async function daSourceHead({ env, daCtx }) { const store = getStore(env, daCtx, source); console.log(`-> HEAD ${store.url.toString()}`); - const response = await store.fetch(store.url, { method: 'HEAD', headers }); + const response = await reachStore(store, store.url, { method: 'HEAD', headers }); + if (!response) return head503(); console.log(`<- HEAD ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return new Response(null, { status: response.status, headers: response.headers }); } @@ -246,6 +275,9 @@ export async function daSourcePost({ req, env, daCtx }) { // the payload is settled, so the only question left is where it goes. A write is the one // operation a wrong guess cannot be walked back from. const source = await resolveContentSource(env, daCtx); + if (source.kind === UNAUTHORIZED) { + return daResp({ body: '', status: source.status, contentType: 'text/plain; charset=utf-8' }); + } if (source.kind === UNKNOWN) { console.warn(`503 POST ${sourcePath}, content source unresolved: ${source.reason}`); return post503(SOURCE_UNRESOLVED_MESSAGE); @@ -256,7 +288,8 @@ export async function daSourcePost({ req, env, daCtx }) { // eslint-disable-next-line no-param-reassign req = new Request(store.url, store.writeInit(bodyContent, authToken)); console.log(`-> ${store.url.toString()}`); - const response = await store.fetch(req); + const response = await reachStore(store, req); + if (!response) return post503(SOURCE_UNREACHABLE_MESSAGE); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } diff --git a/src/storage/content-source.js b/src/storage/content-source.js index f674b050..c44ac57f 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -16,6 +16,7 @@ const TIMEOUT_MS = 5 * 1000; export const SOURCE_BUS = 'sourcebus'; export const LEGACY = 'legacy'; export const UNKNOWN = 'unknown'; +export const UNAUTHORIZED = 'unauthorized'; function unknown(org, site, reason) { console.warn(`[source] ${org}/${site} unknown: ${reason}`); @@ -37,8 +38,9 @@ function unknown(org, site, reason) { * * @param {Object} env worker env, `AEM_API` is the API host and the source-bus prefix * @param {Object} daCtx - * @returns {Promise<{kind: string, base?: string, reason?: string}>} `sourcebus` with the store - * base url, `legacy`, or `unknown` with the reason it could not be answered + * @returns {Promise<{kind: string, base?: string, status?: number, reason?: string}>} `sourcebus` + * with the store base url, `legacy`, `unauthorized` with the status the API gave, or `unknown` + * with the reason it could not be answered */ export default async function resolveContentSource(env, daCtx) { const { org, site, authToken } = daCtx; @@ -66,6 +68,13 @@ export default async function resolveContentSource(env, daCtx) { return unknown(org, site, `${url} failed with ${e.name}: ${e.message}`); } + // an expired or insufficient session is a definite answer, not an unresolved one. Reporting it + // as unresolved answers a retryable 503, and the caller re-tries a session that cannot recover. + if (response.status === 401 || response.status === 403) { + console.warn(`[source] ${org}/${site} not authorized: ${url} answered ${response.status}`); + return { kind: UNAUTHORIZED, status: response.status }; + } + if (response.status !== 200) { return unknown(org, site, `${url} answered ${response.status}`); } diff --git a/src/utils/constants.js b/src/utils/constants.js index 81343083..b6abe7cb 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -53,6 +53,10 @@ export const BRANCH_NOT_FOUND_HTML_MESSAGE = '

Not found: Unable export const SOURCE_UNRESOLVED_HTML_MESSAGE = '

503: Content source unresolved

The store that holds this document could not be determined. Please retry.

'; +export const SOURCE_UNREACHABLE_HTML_MESSAGE = '

503: Content store unreachable

The store that holds this document did not answer. Please retry.

'; + +export const SOURCE_UNREACHABLE_MESSAGE = 'The store that holds this document did not answer, so nothing was written. Please retry.'; + export const SOURCE_UNRESOLVED_MESSAGE = 'The store that holds this document could not be determined, so the write was refused rather than sent to the wrong one. Please retry.'; export const DEFAULT_UNAUTHORIZED_HTML_MESSAGE = '

401: Unauthorized

'; diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index fddab2de..0770b214 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -216,7 +216,9 @@ describe('writing to the store that holds the site', () => { describe('when the store cannot be reached at all', () => { it('answers 503 rather than throwing', async () => { const { daSourcePost, env } = await build({ source: BUS_SOURCE }); - globalThis.fetch = async () => { throw new TypeError('fetch failed'); }; + globalThis.fetch = async () => { + throw new TypeError('fetch failed'); + }; const req = uePost(AT); const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); From a3fe1a79f1aa622f2fa59ecb570ef2b5ab9bdd95 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 10:31:25 +0200 Subject: [PATCH 32/48] test: pin the asset race when the store could not answer the race prefers the published copy at 200, so an image that is published still resolves. where neither answers, a 404 says the image does not exist when the truth is that we could not find out. --- test/routes/source-read.test.js | 61 +++++++++++++++++++++++++++++ test/storage/content-source.test.js | 6 +-- test/storage/store.test.js | 6 +-- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index be10701e..ff4f3103 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -178,6 +178,29 @@ describe('reading with the content source resolved', () => { assert.strictEqual(res.status, 503); assert.strictEqual(await res.text(), ''); }); + + it('asks the caller to retry on a HEAD too', async () => { + const { daSourceHead, env } = await build({ source: UNKNOWN_SOURCE }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.ok(Number(res.headers.get('Retry-After')) > 0); + }); + + // the read refusals are rendered, by the preview iframe and by quick-edit, so they carry a + // body that says what happened rather than an empty page + it('says what happened, on both GET paths', async () => { + const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + const html = authedReq('https://main--site--org.ue.da.live/folder/content'); + const asset = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const htmlBody = await (await daSourceGet({ req: html, env, daCtx: getDaCtx(html) })).text(); + const assetBody = await (await daSourceGet({ req: asset, env, daCtx: getDaCtx(asset) })).text(); + + assert.match(htmlBody, /503/); + assert.match(assetBody, /503/); + }); }); describe('when the caller is not allowed to ask which store', () => { @@ -440,6 +463,44 @@ describe('reading with the content source resolved', () => { }); describe('a media read, which the handlers race against the AEM proxy', () => { + // the store answer is preferred at 200 and the published copy otherwise, so an image that is + // published still resolves during a lookup outage. But when neither answers, a 404 says "this + // image does not exist" where the truth is "we could not find out", so the store's own 503 + // wins over a non-200 from the proxy. + const raced = async (handler, storeStatus, aemStatus) => { + const mod = await esmock(`../../src/handlers/${handler}.js`, { + '../../src/routes/da-admin.js': { + daSourceGet: async () => new Response('', { status: storeStatus }), + daSourceHead: async () => new Response(null, { status: storeStatus }), + }, + '../../src/routes/aem-proxy.js': { + handleAEMProxyRequest: async () => new Response( + aemStatus === 200 ? 'the published bytes' : '', { status: aemStatus }, + ), + }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + return (await mod.default({ req, env: {}, daCtx: getDaCtx(req) })).status; + }; + + ['get', 'head'].forEach((handler) => { + it(`prefers the published copy over an unresolved store on a ${handler.toUpperCase()}`, async () => { + assert.strictEqual(await raced(handler, 503, 200), 200); + }); + + it(`answers 503 rather than 404 when neither answers on a ${handler.toUpperCase()}`, async () => { + assert.strictEqual(await raced(handler, 503, 404), 503); + }); + + it(`still answers 404 when the image is simply absent on a ${handler.toUpperCase()}`, async () => { + assert.strictEqual(await raced(handler, 404, 404), 404); + }); + + it(`still prefers the store at 200 on a ${handler.toUpperCase()}`, async () => { + assert.strictEqual(await raced(handler, 200, 404), 200); + }); + }); + // getHandler races an image read against *.aem.page and takes the proxy answer whenever the // store read is not a 200. So an unresolved source degrades an image to the published copy // rather than breaking the page, and an image cannot be laundered into a write: a POST to a diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index fdd48a68..a3b2f260 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -164,9 +164,9 @@ describe('resolveContentSource', () => { }); describe('when the question could not be answered', () => { - // a 404 is what helix-admin returns when config resolution produced nothing - // (src/sidekick/handler.js: `if (config) { ... } return { status: 404 }`), so it - // means "we do not know", not "legacy" + // a 404 is what the sidekick route returns when config resolution produced nothing + // (helix-api-service and helix-admin both: `if (config) { ... } return { status: 404 }`), so + // it means "we do not know", not "legacy" it('answers unknown on a 404', async () => { stubFetch(() => new Response('', { status: 404 })); diff --git a/test/storage/store.test.js b/test/storage/store.test.js index 38ae23ae..988e6117 100644 --- a/test/storage/store.test.js +++ b/test/storage/store.test.js @@ -162,9 +162,9 @@ describe('getStore', () => { assert.strictEqual(new Headers(l.writeInit('', 'Bearer t').headers).get('Authorization'), 'Bearer t'); }); - // neither store's writes are conditional: only the source bus sets an etag on a read, and a - // marker on the connection uri is minted once per page load while UE saves many times against - // it, so nothing could refresh a version pin + // neither store's writes are conditional. Only the source bus sets an etag on a read, and + // nothing carries it into the save: a marker on the connection uri would be minted once per + // page load while UE saves many times against it, so no version pin could stay fresh. it('sends no precondition to either store', () => { const b = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); const l = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); From 63c43fef028d967871b2ca6c2f31dc25f88c9750 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 10:31:53 +0200 Subject: [PATCH 33/48] fix: let a store that could not answer win the asset race the race prefers the published copy at 200, so a published image still resolves during a lookup outage. taking the proxy's 404 when neither answered said the image does not exist where the truth is that we could not find out. --- src/handlers/get.js | 17 +++++++++++++---- src/handlers/head.js | 15 ++++++++++++--- test/routes/source-read.test.js | 10 +++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/handlers/get.js b/src/handlers/get.js index a472504f..c0291510 100644 --- a/src/handlers/get.js +++ b/src/handlers/get.js @@ -35,11 +35,20 @@ export default async function getHandler({ req, env, daCtx }) { handleAEMProxyRequest({ req, env, daCtx }), ]); + const storeRes = daSourceGetRes.status === 'fulfilled' ? daSourceGetRes.value : undefined; + const aemRes = aemProxyRes.status === 'fulfilled' ? aemProxyRes.value : undefined; + let response; - if (daSourceGetRes.status === 'fulfilled' && daSourceGetRes.value.status === 200) { - response = daSourceGetRes.value; - } else if (aemProxyRes.status === 'fulfilled') { - response = aemProxyRes.value; + if (storeRes?.status === 200) { + response = storeRes; + } else if (aemRes?.status === 200) { + response = aemRes; + } else if (storeRes && storeRes.status >= 500) { + // the store could not answer, so neither can we. Taking the proxy's 404 here would say the + // image does not exist when the truth is that we could not find out. + response = storeRes; + } else if (aemRes) { + response = aemRes; } else { return get404(); } diff --git a/src/handlers/head.js b/src/handlers/head.js index d86b3de4..c24edd9e 100644 --- a/src/handlers/head.js +++ b/src/handlers/head.js @@ -41,10 +41,19 @@ export default async function headHandler({ req, env, daCtx }) { aemHead({ req, env, daCtx }), ]); - if (daSourceHeadRes.status === 'fulfilled' && daSourceHeadRes.value.status === 200) { - return daSourceHeadRes.value; - } + const storeRes = daSourceHeadRes.status === 'fulfilled' ? daSourceHeadRes.value : undefined; const aemResponse = aemHeadRes.status === 'fulfilled' ? aemHeadRes.value : null; + + if (storeRes?.status === 200) { + return storeRes; + } + if (aemResponse?.status === 200) { + return aemResponse; + } + // the store could not answer, so neither can we; the proxy's 404 would claim it does not exist + if (storeRes && storeRes.status >= 500) { + return storeRes; + } if (aemResponse && aemResponse.status < 500) { return aemResponse; } diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index ff4f3103..5ae14191 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -195,8 +195,10 @@ describe('reading with the content source resolved', () => { const html = authedReq('https://main--site--org.ue.da.live/folder/content'); const asset = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); - const htmlBody = await (await daSourceGet({ req: html, env, daCtx: getDaCtx(html) })).text(); - const assetBody = await (await daSourceGet({ req: asset, env, daCtx: getDaCtx(asset) })).text(); + const htmlRes = await daSourceGet({ req: html, env, daCtx: getDaCtx(html) }); + const assetRes = await daSourceGet({ req: asset, env, daCtx: getDaCtx(asset) }); + const htmlBody = await htmlRes.text(); + const assetBody = await assetRes.text(); assert.match(htmlBody, /503/); assert.match(assetBody, /503/); @@ -474,9 +476,7 @@ describe('reading with the content source resolved', () => { daSourceHead: async () => new Response(null, { status: storeStatus }), }, '../../src/routes/aem-proxy.js': { - handleAEMProxyRequest: async () => new Response( - aemStatus === 200 ? 'the published bytes' : '', { status: aemStatus }, - ), + handleAEMProxyRequest: async () => new Response(aemStatus === 200 ? 'the published bytes' : '', { status: aemStatus }), }, }); const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); From a82d1c05c8a5110d5210bdc9a9a101870c39ca0c Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 10:33:57 +0200 Subject: [PATCH 34/48] test: pin the lookup timeout and the half-parsed request guard both survived a mutation pass. without the timeout a config service that never answers holds the request open; with the guard weakened to && a request missing only the site builds a url with "undefined" in it and sends the author token to it. --- test/storage/content-source.test.js | 36 +++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index a3b2f260..f8623735 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -71,6 +71,18 @@ describe('resolveContentSource', () => { assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), 'Bearer t'); }); + // a config service that accepts the connection and never answers would otherwise hold the + // request open for as long as the platform allows + it('gives up on the lookup rather than hanging', async () => { + stubFetch(legacyBody); + + await resolveContentSource(env, daCtx()); + + const { signal } = calls[0].init; + assert.ok(signal, 'the lookup carries an abort signal'); + assert.strictEqual(typeof signal.aborted, 'boolean'); + }); + it('asks anyway when there is no author token', async () => { stubFetch(legacyBody); @@ -242,13 +254,23 @@ describe('resolveContentSource', () => { }); describe('when there is no site to ask about', () => { - it('answers unknown without making a request', async () => { - stubFetch(legacyBody); - - const source = await resolveContentSource(env, daCtx({ org: undefined, site: undefined })); - - assert.strictEqual(source.kind, 'unknown'); - assert.strictEqual(calls.length, 0); + // either one missing is enough: a half-parsed request would otherwise build a url with + // "undefined" in it and send the author's token to it + [ + ['neither', { org: undefined, site: undefined }], + ['no org', { org: undefined }], + ['no site', { site: undefined }], + ['an empty org', { org: '' }], + ['an empty site', { site: '' }], + ].forEach(([what, over]) => { + it(`answers unknown without making a request: ${what}`, async () => { + stubFetch(legacyBody); + + const source = await resolveContentSource(env, daCtx(over)); + + assert.strictEqual(source.kind, 'unknown'); + assert.strictEqual(calls.length, 0); + }); }); }); From 1fcd2cd65965423d6c4553d427d890e7a6306778 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 11:51:37 +0200 Subject: [PATCH 35/48] test: pin the /ping fast path for a source-bus read /ping answers an enrolled site from the Fastly edge in ~37ms without reaching an origin, against ~529ms for the config read. only its yes is usable: its absence conflates legacy, a config that would not resolve, and a site that does not exist. --- test/storage/content-source.test.js | 110 +++++++++++++++++++++++++++- wrangler.toml | 6 +- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index f8623735..b403c74c 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -13,9 +13,11 @@ /* eslint-env mocha */ import assert from 'assert'; -const { default: resolveContentSource } = await import('../../src/storage/content-source.js'); +const { + default: resolveContentSource, fastSourceBus, +} = await import('../../src/storage/content-source.js'); -const env = { AEM_API: 'https://api.aem.live' }; +const env = { AEM_API: 'https://api.aem.live', HLX_ADMIN: 'https://admin.hlx.page' }; const daCtx = (over = {}) => ({ org: 'org', site: 'site', ref: 'main', authToken: 'Bearer t', ...over, @@ -320,3 +322,107 @@ describe('resolveContentSource', () => { }); }); }); + +describe('fastSourceBus', () => { + afterEach(() => { + delete globalThis.fetch; + }); + + // /ping answers an enrolled site from the Fastly edge dictionary in ~37ms without reaching an + // origin, where the config read is ~529ms and always reaches one. Its `true` is positive + // evidence; its absence conflates legacy, a config that would not resolve, and a site that does + // not exist, so only the yes is usable. + const ping = (headers = {}, status = 200) => new Response('', { status, headers }); + + it('asks /ping on the admin host', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + await fastSourceBus(env, daCtx()); + + assert.strictEqual(calls.length, 1); + assert.strictEqual(calls[0].url, 'https://admin.hlx.page/ping/org/site'); + }); + + // /ping is exempt from authorize() in helix-admin and answers the same with or without a token + it('sends no token, since /ping does not read one', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + await fastSourceBus(env, daCtx()); + + assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), null); + }); + + it('gives up rather than hanging', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + await fastSourceBus(env, daCtx()); + + assert.ok(calls[0].init.signal, 'the probe carries an abort signal'); + }); + + describe('when /ping says the site is upgraded', () => { + it('answers sourcebus', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + assert.strictEqual((await fastSourceBus(env, daCtx())).kind, 'sourcebus'); + }); + + // helix-api-service parses org and site out of the source url and 400s unless both match the + // request's own (src/contentproxy/source/utils.js, "only allow source bus from the same org + // and site"), so this is the only base a source-bus site can legally have + it('builds the only base that org and site can legally have', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + const source = await fastSourceBus(env, daCtx()); + + assert.strictEqual(source.base, 'https://api.aem.live/org/sites/site/source'); + }); + + it('builds it on AEM_API, so stage moves it', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + const source = await fastSourceBus({ ...env, AEM_API: 'https://api.stage.example' }, daCtx()); + + assert.strictEqual(source.base, 'https://api.stage.example/org/sites/site/source'); + }); + }); + + describe('when /ping does not say so', () => { + [ + ['the header is absent', {}, 200], + ['the header is false', { 'x-api-upgrade-available': 'false' }, 200], + ['the header is empty', { 'x-api-upgrade-available': '' }, 200], + ['the header is TRUE in capitals', { 'x-api-upgrade-available': 'TRUE' }, 200], + ['the status is 404', { 'x-api-upgrade-available': 'true' }, 404], + ['the status is 405', {}, 405], + ['the status is 500', {}, 500], + ].forEach(([what, headers, status]) => { + it(`answers undefined: ${what}`, async () => { + stubFetch(() => ping(headers, status)); + + assert.strictEqual(await fastSourceBus(env, daCtx()), undefined); + }); + }); + + it('answers undefined when the probe throws', async () => { + stubFetch(() => { + throw new TypeError('fetch failed'); + }); + + assert.strictEqual(await fastSourceBus(env, daCtx()), undefined); + }); + + it('answers undefined without asking when there is no site', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + assert.strictEqual(await fastSourceBus(env, daCtx({ site: undefined })), undefined); + assert.strictEqual(calls.length, 0); + }); + + it('answers undefined when the admin host is unusable', async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + assert.strictEqual(await fastSourceBus({ ...env, HLX_ADMIN: 'nope' }, daCtx()), undefined); + }); + }); +}); diff --git a/wrangler.toml b/wrangler.toml index 713c655e..799ca37e 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,18 +2,18 @@ name = "da-ue" main = "src/index.js" compatibility_date = "2023-11-21" -vars = { UE_HOST = "ue.da.live", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live" } +vars = { UE_HOST = "ue.da.live", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live", HLX_ADMIN = "https://admin.hlx.page" } services = [{ binding = "daadmin", service = "da-admin" }] [dev] port = 4712 [env.dev] -vars = { ENVIRONMENT = "dev", UE_HOST = "localhost:4712", UE_SERVICE = "https://localhost:8000", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live" } +vars = { ENVIRONMENT = "dev", UE_HOST = "localhost:4712", UE_SERVICE = "https://localhost:8000", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live", HLX_ADMIN = "https://admin.hlx.page" } services = [{ binding = "daadmin", service = "da-admin-local" }] [env.stage] -vars = { ENVIRONMENT = "stage", UE_HOST = "stage-ue.da.live", UE_SERVICE = "https://universal-editor-service-dev.adobe.io", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live" } +vars = { ENVIRONMENT = "stage", UE_HOST = "stage-ue.da.live", UE_SERVICE = "https://universal-editor-service-dev.adobe.io", DA_ADMIN = "https://admin.da.live", AEM_API = "https://api.aem.live", HLX_ADMIN = "https://admin.hlx.page" } services = [{ binding = "daadmin", service = "da-admin" }] [env.stage.observability] From 0ba20981e109f120632dcda5bdf8fda763735556 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 11:52:52 +0200 Subject: [PATCH 36/48] test: pin the fast path wired into reads, and never into writes the fast answer is trusted only when it produced content, so a yes that finds nothing falls back to the config read rather than becoming the starter template. /ping reads no token, so the store's 401 has to serve the da:401 shell the extension recovers from. --- src/storage/content-source.js | 43 +++++++++ test/routes/source-read.test.js | 151 +++++++++++++++++++++++++++++++- 2 files changed, 192 insertions(+), 2 deletions(-) diff --git a/src/storage/content-source.js b/src/storage/content-source.js index c44ac57f..9961fd14 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -12,12 +12,55 @@ const LEGACY_PREFIX = 'https://content.da.live/'; const TIMEOUT_MS = 5 * 1000; +const UPGRADE_HEADER = 'x-api-upgrade-available'; export const SOURCE_BUS = 'sourcebus'; export const LEGACY = 'legacy'; export const UNKNOWN = 'unknown'; export const UNAUTHORIZED = 'unauthorized'; +function sourceBusBase(env, org, site) { + return `${env.AEM_API?.replace(/\/$/, '')}/${org}/sites/${site}/source`; +} + +/** + * Asks `/ping` whether a site is on the source bus, for a read. + * + * The Fastly edge answers an enrolled site from a dictionary in ~37ms without reaching an origin, + * against ~529ms for the config read. Only the yes is usable: helix-admin sets the header when + * config resolution succeeded and named the API, so its absence covers a legacy site, a config + * that would not resolve, and a site that does not exist alike. + * + * The base is built rather than read, because `/ping` returns a header and no url. + * helix-api-service parses org and site out of a source url and refuses one that names another + * site (`src/contentproxy/source/utils.js`, "only allow source bus from the same org and site"), + * so this is the only base the site can legally have. + * + * @returns {Promise<{kind: string, base: string}|undefined>} undefined whenever `/ping` did not + * say yes, which leaves the config read to answer + */ +export async function fastSourceBus(env, daCtx) { + const { org, site } = daCtx; + if (!org || !site) return undefined; + + let url; + try { + url = new URL(`/ping/${org}/${site}`, env.HLX_ADMIN); + } catch (e) { + return undefined; + } + + try { + // no token: /ping is exempt from authorize() and answers the same either way + const response = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }); + if (response.status !== 200) return undefined; + if (response.headers.get(UPGRADE_HEADER) !== 'true') return undefined; + } catch (e) { + return undefined; + } + return { kind: SOURCE_BUS, base: sourceBusBase(env, org, site) }; +} + function unknown(org, site, reason) { console.warn(`[source] ${org}/${site} unknown: ${reason}`); return { kind: UNKNOWN, reason }; diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 5ae14191..adad35ed 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -37,7 +37,7 @@ const build = async (overrides = {}) => { // 'headHtml' in overrides rather than a destructured default, so passing // `{ headHtml: undefined }` really does simulate a missing head.html const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; - const seen = { bus: [], legacy: [], ue: 0 }; + const seen = { bus: [], legacy: [], ue: 0, ping: 0, config: 0 }; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); seen.bus.push({ url: request.url, method: request.method, headers: request.headers }); @@ -56,10 +56,18 @@ const build = async (overrides = {}) => { }; const mod = await esmock('../../src/routes/da-admin.js', { '../../src/storage/content-source.js': { - default: async () => source, + default: async () => { + seen.config += 1; + return source; + }, + fastSourceBus: async () => { + seen.ping += 1; + return overrides.fast; + }, SOURCE_BUS: 'sourcebus', LEGACY: 'legacy', UNKNOWN: 'unknown', + UNAUTHORIZED: 'unauthorized', }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({}), @@ -83,6 +91,145 @@ afterEach(() => { delete globalThis.fetch; }); +describe('the /ping fast path on a read', () => { + const FAST = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; + + it('serves the source-bus document without waiting on the config read', async () => { + const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 200); + assert.strictEqual(seen.ping, 1); + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); + }); + + it('takes it on a non-html read too, which is the per-image path', async () => { + const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); + const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 200); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Media/holiday.PNG'); + }); + + it('takes it on a HEAD', async () => { + const { daSourceHead, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 200); + assert.strictEqual(seen.bus.length, 1); + }); + + // /ping is trusted only when it produced content. its absence conflates legacy with a config + // that would not resolve, and the CF port of the edge hardcodes the yes, so a yes that finds + // nothing must not become the starter template for the author to save over + it('falls back to the config read when the fast store has nothing', async () => { + const { daSourceGet, env, seen } = await build({ + source: LEGACY_SOURCE, + fast: FAST, + bus: () => new Response('', { status: 404 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 200); + assert.strictEqual(await res.text(), 'from da-admin'); + assert.strictEqual(seen.legacy.length, 1); + }); + + it('falls back when the fast store refuses', async () => { + const { daSourceGet, env, seen } = await build({ + source: LEGACY_SOURCE, + fast: FAST, + bus: () => new Response('', { status: 403 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.legacy.length, 1); + }); + + it('falls back when the fast store cannot be reached', async () => { + const { daSourceGet, env, seen } = await build({ + source: LEGACY_SOURCE, + fast: FAST, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.legacy.length, 1); + }); + + it('does not take it when /ping did not say yes', async () => { + const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE, fast: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.ping, 1); + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy.length, 1); + }); + + // a save is one request where 460ms does not matter, and a wrong store on a write cannot be + // walked back + it('is never taken on a write', async () => { + const { daSourcePost, env, seen } = await build({ source: LEGACY_SOURCE, fast: FAST }); + const body = new FormData(); + body.set('data', new File(['

x

'], 'c.html', { type: 'text/html' })); + const req = new Request('https://main--site--org.ue.da.live/folder/content', { + method: 'POST', body, headers: { Authorization: 'Bearer t' }, + }); + + await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(seen.ping, 0); + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy.length, 1); + }); + + // /ping reads no token, so on the fast path the store is the first thing to see one. the + // authorbus extension recovers off the da:401 meta and never reads the status, and both stores + // answer 401 with an empty body. + it('serves the da:401 shell when the store refuses the token on an html read', async () => { + const { daSourceGet, env } = await build({ + source: UNKNOWN_SOURCE, + fast: FAST, + bus: () => new Response('', { status: 401 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 401); + assert.match(await res.text(), /content="da:401"/); + }); + + it('passes a store 401 through bare on a non-html read, which renders nothing', async () => { + const { daSourceGet, env } = await build({ + source: UNKNOWN_SOURCE, + fast: FAST, + bus: () => new Response('', { status: 401 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 401); + assert.strictEqual(await res.text(), ''); + }); +}); + describe('reading with the content source resolved', () => { describe('an html read on a source-bus site', () => { it('reads from the base the config named', async () => { From cf427c1f8624bc9ec61987510223709b1fb9a4a4 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 11:54:01 +0200 Subject: [PATCH 37/48] feat: take /ping's fast answer on a read an enrolled site answers from the Fastly edge in ~37ms where the config read is ~529ms, and a previewed page makes one lookup per image. both run at once, since /ping teaches a legacy site nothing and would otherwise be paid in series. the fast answer is trusted on a 200, or on a refusal, which is about the token rather than about which store. anything else falls back to the config read, so a yes that found nothing cannot become the starter template. writes never take it: one request where 460ms does not matter, and the one place a wrong store cannot be walked back. --- src/routes/da-admin.js | 79 +++++++++++++++++++++------------ test/routes/source-read.test.js | 25 ++++------- 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index fd4fb1ff..a00cc8cc 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -34,7 +34,7 @@ import { UNAUTHORIZED_HTML_MESSAGE, } from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; -import resolveContentSource, { UNAUTHORIZED, UNKNOWN } from '../storage/content-source.js'; +import resolveContentSource, { fastSourceBus, UNAUTHORIZED, UNKNOWN } from '../storage/content-source.js'; import getStore from '../storage/store.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; @@ -101,6 +101,38 @@ async function reachStore(store, input, init) { } } +/** + * Reads from the store that holds the site, taking `/ping`'s fast answer when it produced content. + * + * The two lookups run at once: `/ping` answers an enrolled site from the Fastly edge in ~37ms, + * where the config read is ~529ms, and a legacy site learns nothing from `/ping` so it would + * otherwise pay both in series. The fast answer is trusted only on a 200, because `/ping` cannot + * say "legacy" and a yes that found nothing would otherwise become the starter template. + * + * @returns {Promise<{source: Object, response?: Response}>} `response` is absent when the store + * could not be reached, and when `source.kind` is unauthorized or unknown + */ +async function readSource(env, daCtx, init) { + const config = resolveContentSource(env, daCtx); + const fast = await fastSourceBus(env, daCtx); + if (fast) { + const store = getStore(env, daCtx, fast); + console.log(`-> ${init.method} ${store.url.toString()} (fast)`); + const response = await reachStore(store, store.url, init); + if (response?.status === 200) return { source: fast, response }; + // a refusal is about the token, not about which store, so the config read would repeat it + if (response?.status === 401 || response?.status === 403) return { source: fast, response }; + } + + const source = await config; + if (source.kind === UNAUTHORIZED || source.kind === UNKNOWN) return { source }; + + const store = getStore(env, daCtx, source); + console.log(`-> ${init.method} ${store.url.toString()}`); + const response = await reachStore(store, store.url, init); + return { source, response }; +} + export async function daSourceGet({ req, env, daCtx }) { const { ext, authToken } = daCtx; @@ -121,29 +153,28 @@ export async function daSourceGet({ req, env, daCtx }) { headers.set('Authorization', authToken); if (ext !== 'html') { - // for non-HTML files, simply proxy the request without processing - const source = await resolveContentSource(env, daCtx); + // for non-HTML files, simply proxy the request without processing. A refusal is passed on as + // itself: nothing renders an image, so the da:401 shell would only corrupt it. + const { source, response } = await readSource(env, daCtx, { method: 'GET', headers }); if (source.kind === UNAUTHORIZED) { - return daResp({ body: UNAUTHORIZED_HTML_MESSAGE, status: source.status, contentType: 'text/html' }); + return daResp({ body: '', status: source.status, contentType: 'text/plain; charset=utf-8' }); } if (source.kind === UNKNOWN) { console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); } - const store = getStore(env, daCtx, source); - console.log(`-> ${store.url.toString()}`); - const response = await reachStore(store, store.url, { method: 'GET', headers }); if (!response) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); - console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); + console.log(`<- ${daCtx.sourcePath}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } // the store lookup costs a round trip, so it runs alongside head.html rather than after it const aemCtx = getAemCtx(env, daCtx); - const [headHtml, source] = await Promise.all([ + const [headHtml, read] = await Promise.all([ getAEMHtml(aemCtx, '/head.html'), - resolveContentSource(env, daCtx), + readSource(env, daCtx, { method: 'GET', headers }), ]); + const { source, response: sourceResp } = read; if (!headHtml) { // quick-edit still needs a working shell (with the import map) so the editor // can load into this page, even when the AEM branch doesn't exist yet. @@ -159,19 +190,15 @@ export async function daSourceGet({ req, env, daCtx }) { console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); } - - // get the content from the store that holds it - const store = getStore(env, daCtx, source); - - // eslint-disable-next-line no-param-reassign - req = new Request(store.url, { - method: 'GET', - headers, - }); - console.log(`-> ${store.url.toString()}`); - const sourceResp = await reachStore(store, req); if (!sourceResp) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); - console.log(`<- ${store.url.toString()}. ${sourceResp.status} ${sourceResp.statusText}`, { status: sourceResp.status, statusText: sourceResp.statusText }); + console.log(`<- ${daCtx.sourcePath}. ${sourceResp.status} ${sourceResp.statusText}`, { status: sourceResp.status, statusText: sourceResp.statusText }); + + // the store is the first thing to see the token when the fast path skipped the config read, and + // the authorbus extension recovers off the da:401 meta rather than the status, so a refusal from + // the store gets the same shell the config read would have produced + if (sourceResp.status === 401 || sourceResp.status === 403) { + return daResp({ body: UNAUTHORIZED_HTML_MESSAGE, status: sourceResp.status, contentType: 'text/html' }); + } // only a 404 means "this document is not here". Composing the starter template over anything // else hands the author a blank page to save over a document that exists. @@ -221,7 +248,7 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const source = await resolveContentSource(env, daCtx); + const { source, response } = await readSource(env, daCtx, { method: 'HEAD', headers }); if (source.kind === UNAUTHORIZED) { return new Response(null, { status: source.status }); } @@ -229,12 +256,8 @@ export async function daSourceHead({ env, daCtx }) { console.warn(`503 HEAD ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); return head503(); } - - const store = getStore(env, daCtx, source); - console.log(`-> HEAD ${store.url.toString()}`); - const response = await reachStore(store, store.url, { method: 'HEAD', headers }); if (!response) return head503(); - console.log(`<- HEAD ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); + console.log(`<- HEAD ${daCtx.sourcePath}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return new Response(null, { status: response.status, headers: response.headers }); } diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index adad35ed..a0558073 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -37,7 +37,9 @@ const build = async (overrides = {}) => { // 'headHtml' in overrides rather than a destructured default, so passing // `{ headHtml: undefined }` really does simulate a missing head.html const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; - const seen = { bus: [], legacy: [], ue: 0, ping: 0, config: 0 }; + const seen = { + bus: [], legacy: [], ue: 0, ping: 0, config: 0, + }; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); seen.bus.push({ url: request.url, method: request.method, headers: request.headers }); @@ -144,24 +146,13 @@ describe('the /ping fast path on a read', () => { assert.strictEqual(seen.legacy.length, 1); }); - it('falls back when the fast store refuses', async () => { - const { daSourceGet, env, seen } = await build({ - source: LEGACY_SOURCE, - fast: FAST, - bus: () => new Response('', { status: 403 }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(seen.legacy.length, 1); - }); - it('falls back when the fast store cannot be reached', async () => { const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE, fast: FAST, - bus: () => { throw new TypeError('fetch failed'); }, + bus: () => { + throw new TypeError('fetch failed'); + }, }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); @@ -203,7 +194,7 @@ describe('the /ping fast path on a read', () => { // answer 401 with an empty body. it('serves the da:401 shell when the store refuses the token on an html read', async () => { const { daSourceGet, env } = await build({ - source: UNKNOWN_SOURCE, + source: DENIED_SOURCE, fast: FAST, bus: () => new Response('', { status: 401 }), }); @@ -217,7 +208,7 @@ describe('the /ping fast path on a read', () => { it('passes a store 401 through bare on a non-html read, which renders nothing', async () => { const { daSourceGet, env } = await build({ - source: UNKNOWN_SOURCE, + source: DENIED_SOURCE, fast: FAST, bus: () => new Response('', { status: 401 }), }); From ddb40b7119f51772dcd6d6f83eeef0b02d846caa Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 12:44:56 +0200 Subject: [PATCH 38/48] test: pin that a fast answer needs a base that parses --- test/storage/content-source.test.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index b403c74c..c48721e2 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -424,5 +424,23 @@ describe('fastSourceBus', () => { assert.strictEqual(await fastSourceBus({ ...env, HLX_ADMIN: 'nope' }, daCtx()), undefined); }); + + // getStore builds the store url from this base and its `new URL` is unguarded, so a base that + // cannot parse throws out of `worker.fetch` before withCorsHeaders runs: an opaque 500 with no + // CORS on a document read, and a 404 on the raced image path where allSettled swallows it. The + // config read answers unknown for the same env, so the caller gets a 503 instead. + [ + ['AEM_API is unset', undefined], + ['AEM_API has no scheme', 'api.aem.live'], + ['AEM_API is protocol-relative', '//api.aem.live'], + ['AEM_API has a trailing space', 'https://api.aem.live '], + ].forEach(([what, AEM_API]) => { + it(`answers undefined without asking when ${what}`, async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); + + assert.strictEqual(await fastSourceBus({ ...env, AEM_API }, daCtx()), undefined); + assert.strictEqual(calls.length, 0); + }); + }); }); }); From 010823fc4ba1faff546819e772fbf95729125825 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 12:45:24 +0200 Subject: [PATCH 39/48] fix: refuse a fast answer whose base is not a url --- src/storage/content-source.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/storage/content-source.js b/src/storage/content-source.js index 9961fd14..70110e87 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -19,8 +19,14 @@ export const LEGACY = 'legacy'; export const UNKNOWN = 'unknown'; export const UNAUTHORIZED = 'unauthorized'; +// getStore appends the path to this and calls `new URL` on the result, where a throw escapes +// `worker.fetch` before CORS headers are added, so an unusable AEM_API is refused here instead. function sourceBusBase(env, org, site) { - return `${env.AEM_API?.replace(/\/$/, '')}/${org}/sites/${site}/source`; + try { + return new URL(`${env.AEM_API?.replace(/\/$/, '')}/${org}/sites/${site}/source`).toString(); + } catch (e) { + return undefined; + } } /** @@ -43,6 +49,10 @@ export async function fastSourceBus(env, daCtx) { const { org, site } = daCtx; if (!org || !site) return undefined; + // built before the probe, since a base that cannot be used makes the answer unusable too + const base = sourceBusBase(env, org, site); + if (!base) return undefined; + let url; try { url = new URL(`/ping/${org}/${site}`, env.HLX_ADMIN); @@ -58,7 +68,7 @@ export async function fastSourceBus(env, daCtx) { } catch (e) { return undefined; } - return { kind: SOURCE_BUS, base: sourceBusBase(env, org, site) }; + return { kind: SOURCE_BUS, base }; } function unknown(org, site, reason) { From b9e4053eff6e3b6e6b83d50fcf54739c35a34b73 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 3 Aug 2026 12:47:21 +0200 Subject: [PATCH 40/48] test: pin the fast path's refusal branch, its method, and a dotted directory --- test/routes/source-read.test.js | 26 +++++++++++++++++++++++--- test/storage/store.test.js | 1 + 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index a0558073..961e129c 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -126,6 +126,7 @@ describe('the /ping fast path on a read', () => { assert.strictEqual(res.status, 200); assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.bus[0].method, 'HEAD'); }); // /ping is trusted only when it produced content. its absence conflates legacy with a config @@ -191,10 +192,11 @@ describe('the /ping fast path on a read', () => { // /ping reads no token, so on the fast path the store is the first thing to see one. the // authorbus extension recovers off the da:401 meta and never reads the status, and both stores - // answer 401 with an empty body. + // answer 401 with an empty body. the config answer is legacy in these two, so the 401 can only + // have come from the fast store: with a denied config answer they pass either way. it('serves the da:401 shell when the store refuses the token on an html read', async () => { const { daSourceGet, env } = await build({ - source: DENIED_SOURCE, + source: LEGACY_SOURCE, fast: FAST, bus: () => new Response('', { status: 401 }), }); @@ -208,7 +210,7 @@ describe('the /ping fast path on a read', () => { it('passes a store 401 through bare on a non-html read, which renders nothing', async () => { const { daSourceGet, env } = await build({ - source: DENIED_SOURCE, + source: LEGACY_SOURCE, fast: FAST, bus: () => new Response('', { status: 401 }), }); @@ -219,6 +221,24 @@ describe('the /ping fast path on a read', () => { assert.strictEqual(res.status, 401); assert.strictEqual(await res.text(), ''); }); + + // any other store answer says nothing about which store holds the site, so the config read + // decides. keeping the fast source there would answer 503 to a session that cannot recover by + // retrying, since the token is the thing that needs replacing + it('lets the config read answer when the fast store neither served nor refused', async () => { + const { daSourceGet, env, seen } = await build({ + source: DENIED_SOURCE, + fast: FAST, + bus: () => new Response('', { status: 500 }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 401); + assert.match(await res.text(), /content="da:401"/); + assert.strictEqual(seen.config, 1); + }); }); describe('reading with the content source resolved', () => { diff --git a/test/storage/store.test.js b/test/storage/store.test.js index 988e6117..2807843b 100644 --- a/test/storage/store.test.js +++ b/test/storage/store.test.js @@ -36,6 +36,7 @@ describe('sourceBusPath', () => { ['/Sub-Folder/', '/Sub-Folder/index.html', 'keeps directory case on a directory index'], ['/folder/Content', '/folder/content.html', 'lowercases a stem that had no extension'], ['/folder/content.plain.html', '/folder/content.plain.html', 'treats only the last dot as the extension'], + ['/2026.q1/report', '/2026.q1/report.html', 'ignores a dot in a directory segment'], ]; cases.forEach(([path, expected, what]) => { From a5a72dfa379a6429bac3dc8501bdf3eeb2f1eee2 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Thu, 6 Aug 2026 18:59:11 +0200 Subject: [PATCH 41/48] fix: canonical source-bus paths and fewer local catches sourceBusPath is gone, both stores take daCtx.sourcePath. env vars are used raw with one backstop in worker.fetch. the fast path returns any store answer but 404, and getStore exposes write() rather than writeInit. --- src/index.js | 35 +++++---- src/routes/da-admin.js | 31 ++++---- src/storage/content-source.js | 38 ++-------- src/storage/store.js | 42 ++++------- test/routes/da-admin.test.js | 2 + test/routes/source-read.test.js | 40 +++++----- test/routes/source-write.test.js | 6 +- test/storage/content-source.test.js | 42 ----------- test/storage/store.test.js | 113 ++++++++++++---------------- test/utils/daCtx.test.js | 12 +++ 10 files changed, 139 insertions(+), 222 deletions(-) diff --git a/src/index.js b/src/index.js index eb5d1e3c..e6367675 100644 --- a/src/index.js +++ b/src/index.js @@ -49,21 +49,26 @@ export default { const daCtx = getDaCtx(req); let resp; - switch (req.method) { - case 'OPTIONS': - resp = await optionsHandler({ req }); - break; - case 'HEAD': - resp = await headHandler({ req, env, daCtx }); - break; - case 'GET': - resp = await getHandler({ req, env, daCtx }); - break; - case 'POST': - resp = await postHandlers({ req, env, daCtx }); - break; - default: - resp = unknownHandler(); + try { + switch (req.method) { + case 'OPTIONS': + resp = await optionsHandler({ req }); + break; + case 'HEAD': + resp = await headHandler({ req, env, daCtx }); + break; + case 'GET': + resp = await getHandler({ req, env, daCtx }); + break; + case 'POST': + resp = await postHandlers({ req, env, daCtx }); + break; + default: + resp = unknownHandler(); + } + } catch (e) { + console.error(`500 ${req.method} ${url.pathname}: ${e.name}: ${e.message}`, e); + resp = new Response(null, { status: 500 }); } return withCorsHeaders(resp, req); }, diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index a00cc8cc..c628ec04 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -88,13 +88,10 @@ async function getPageTemplate(env, daCtx, aemCtx) { /** * Sends a request to a store and answers 503 when it could not be reached at all. - * - * A throw escaping a handler reaches withCorsHeaders, which reads `response.headers` and throws - * again, so the caller gets an opaque 500 with no CORS headers on it. */ -async function reachStore(store, input, init) { +async function reachStore(store, send) { try { - return await store.fetch(input, init); + return await send(); } catch (e) { console.warn(`503 ${store.url}, the store could not be reached: ${e.name}: ${e.message}`); return undefined; @@ -102,12 +99,16 @@ async function reachStore(store, input, init) { } /** - * Reads from the store that holds the site, taking `/ping`'s fast answer when it produced content. + * Reads from the store that holds the site, taking `/ping`'s fast answer when the store replied. * * The two lookups run at once: `/ping` answers an enrolled site from the Fastly edge in ~37ms, * where the config read is ~529ms, and a legacy site learns nothing from `/ping` so it would - * otherwise pay both in series. The fast answer is trusted only on a 200, because `/ping` cannot - * say "legacy" and a yes that found nothing would otherwise become the starter template. + * otherwise pay both in series. + * + * A 404 is the one answer that falls through to the config read, since a wrong yes is what + * produces one and the starter template would then be composed over a document that exists in the + * other store. Any other status is about the store rather than about which store, so returning it + * saves fetching the same url twice. * * @returns {Promise<{source: Object, response?: Response}>} `response` is absent when the store * could not be reached, and when `source.kind` is unauthorized or unknown @@ -118,10 +119,8 @@ async function readSource(env, daCtx, init) { if (fast) { const store = getStore(env, daCtx, fast); console.log(`-> ${init.method} ${store.url.toString()} (fast)`); - const response = await reachStore(store, store.url, init); - if (response?.status === 200) return { source: fast, response }; - // a refusal is about the token, not about which store, so the config read would repeat it - if (response?.status === 401 || response?.status === 403) return { source: fast, response }; + const response = await reachStore(store, () => store.fetch(store.url, init)); + if (response && response.status !== 404) return { source: fast, response }; } const source = await config; @@ -129,7 +128,7 @@ async function readSource(env, daCtx, init) { const store = getStore(env, daCtx, source); console.log(`-> ${init.method} ${store.url.toString()}`); - const response = await reachStore(store, store.url, init); + const response = await reachStore(store, () => store.fetch(store.url, init)); return { source, response }; } @@ -306,12 +305,10 @@ export async function daSourcePost({ req, env, daCtx }) { return post503(SOURCE_UNRESOLVED_MESSAGE); } - // the two stores take the document in different shapes, so the store builds its own request + // the two stores take the document in different shapes, so the store sends its own request const store = getStore(env, daCtx, source); - // eslint-disable-next-line no-param-reassign - req = new Request(store.url, store.writeInit(bodyContent, authToken)); console.log(`-> ${store.url.toString()}`); - const response = await reachStore(store, req); + const response = await reachStore(store, () => store.write(bodyContent, authToken)); if (!response) return post503(SOURCE_UNREACHABLE_MESSAGE); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; diff --git a/src/storage/content-source.js b/src/storage/content-source.js index 70110e87..cdaaeed3 100644 --- a/src/storage/content-source.js +++ b/src/storage/content-source.js @@ -19,16 +19,6 @@ export const LEGACY = 'legacy'; export const UNKNOWN = 'unknown'; export const UNAUTHORIZED = 'unauthorized'; -// getStore appends the path to this and calls `new URL` on the result, where a throw escapes -// `worker.fetch` before CORS headers are added, so an unusable AEM_API is refused here instead. -function sourceBusBase(env, org, site) { - try { - return new URL(`${env.AEM_API?.replace(/\/$/, '')}/${org}/sites/${site}/source`).toString(); - } catch (e) { - return undefined; - } -} - /** * Asks `/ping` whether a site is on the source bus, for a read. * @@ -49,26 +39,17 @@ export async function fastSourceBus(env, daCtx) { const { org, site } = daCtx; if (!org || !site) return undefined; - // built before the probe, since a base that cannot be used makes the answer unusable too - const base = sourceBusBase(env, org, site); - if (!base) return undefined; - - let url; + const url = new URL(`/ping/${org}/${site}`, env.HLX_ADMIN); try { - url = new URL(`/ping/${org}/${site}`, env.HLX_ADMIN); - } catch (e) { - return undefined; - } - - try { - // no token: /ping is exempt from authorize() and answers the same either way const response = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }); if (response.status !== 200) return undefined; if (response.headers.get(UPGRADE_HEADER) !== 'true') return undefined; } catch (e) { + console.warn(`[source] ${org}/${site} ping failed: ${e.name}: ${e.message}`); return undefined; } - return { kind: SOURCE_BUS, base }; + + return { kind: SOURCE_BUS, base: `${env.AEM_API}/${org}/sites/${site}/source` }; } function unknown(org, site, reason) { @@ -103,14 +84,7 @@ export default async function resolveContentSource(env, daCtx) { return unknown(org, site, 'no org or site in the request'); } - const api = env.AEM_API?.replace(/\/$/, ''); - let url; - try { - url = new URL(`/${org}/sites/${site}/sidekick`, api); - } catch (e) { - return unknown(org, site, `AEM_API is not a url: ${e.message}`); - } - + const url = new URL(`/${org}/sites/${site}/sidekick`, env.AEM_API); const headers = new Headers(); if (authToken) headers.set('Authorization', authToken); @@ -143,7 +117,7 @@ export default async function resolveContentSource(env, daCtx) { if (typeof sourceUrl !== 'string') { return unknown(org, site, `${url} named no content source`); } - if (sourceUrl.startsWith(`${api}/`)) { + if (sourceUrl.startsWith(`${env.AEM_API}/`)) { return { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; } if (sourceUrl.startsWith(LEGACY_PREFIX)) { diff --git a/src/storage/store.js b/src/storage/store.js index 335d131e..95cc359f 100644 --- a/src/storage/store.js +++ b/src/storage/store.js @@ -11,27 +11,6 @@ */ import { SOURCE_BUS } from './content-source.js'; -/** - * Restores the case the source bus stores a file under. - * - * helix-api-service sanitizes the basename and nothing else: `computePaths` pops the filename, - * runs `sanitizeName` on it and recombines the directory segments untouched. The extension comes - * back verbatim too. `daCtx.sourcePath` lowercases the whole path, which is what da-admin wants - * and what the source bus 404s on. - * - * @param {Object} daCtx - * @returns {string} the store path, directory and extension in the case they were requested - */ -export function sourceBusPath({ path, sourcePath }) { - const dirEnd = path.lastIndexOf('/'); - const base = sourcePath.slice(sourcePath.lastIndexOf('/') + 1); - const baseDot = base.lastIndexOf('.'); - const requestedDot = path.lastIndexOf('.'); - // an extension the request carried keeps its case; the `.html` we appended does not have one - const ext = requestedDot > dirEnd ? path.slice(requestedDot) : base.slice(baseDot); - return `${path.slice(0, dirEnd)}/${base.slice(0, baseDot)}${ext}`; -} - /** * Picks the store for a request, the way to reach it, and the shape it takes a write in. * @@ -39,7 +18,8 @@ export function sourceBusPath({ path, sourcePath }) { * fetch cannot serve both. They also differ on the write body: helix-api-service reads the raw * request body and types it from the path extension, parsing no form data anywhere, while * da-admin takes the document as a `data` form part. Handing either the other's shape stores - * something other than the document and answers 201. + * something other than the document and answers 201. `write` sends the document so the caller + * never assembles either shape. * * @param {Object} env worker env * @param {Object} daCtx @@ -49,24 +29,30 @@ export default function getStore(env, daCtx, source) { const { org, site, sourcePath } = daCtx; if (source.kind === SOURCE_BUS) { + const url = new URL(`${source.base}${sourcePath}`); return { - url: new URL(`${source.base}${sourceBusPath(daCtx)}`), + url, fetch: (input, init) => fetch(input, init), - writeInit: (html, authToken) => ({ + write: (html, authToken) => fetch(new Request(url, { method: 'POST', body: html, headers: { Authorization: authToken, 'Content-Type': 'text/html' }, - }), + })), }; } + const url = new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); return { - url: new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN), + url, fetch: (input, init) => env.daadmin.fetch(input, init), - writeInit: (html, authToken) => { + write: (html, authToken) => { const body = new FormData(); body.set('data', new Blob([html], { type: 'text/html' })); - return { method: 'POST', body, headers: { Authorization: authToken } }; + return env.daadmin.fetch(new Request(url, { + method: 'POST', + body, + headers: { Authorization: authToken }, + })); }, }; } diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index 887b6974..cef2dbb6 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -33,6 +33,7 @@ const recorder = () => { const env = { DA_ADMIN: 'https://admin.da.live', AEM_API: 'https://api.aem.live', + HLX_ADMIN: 'https://admin.hlx.page', daadmin: { fetch: async (input) => { fetched.push(input instanceof Request ? input.url : input.href); @@ -103,6 +104,7 @@ describe('daSourceGet', () => { return (await esmock('../../src/routes/da-admin.js', { '../../src/storage/content-source.js': { default: async () => ({ kind: 'legacy' }), + fastSourceBus: async () => undefined, SOURCE_BUS: 'sourcebus', LEGACY: 'legacy', UNKNOWN: 'unknown', diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 961e129c..a5950e4f 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -110,12 +110,12 @@ describe('the /ping fast path on a read', () => { it('takes it on a non-html read too, which is the per-image path', async () => { const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); - const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); + const req = authedReq('https://main--site--org.ue.da.live/media/holiday.png'); const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); assert.strictEqual(res.status, 200); - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Media/holiday.PNG'); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/media/holiday.png'); }); it('takes it on a HEAD', async () => { @@ -222,22 +222,23 @@ describe('the /ping fast path on a read', () => { assert.strictEqual(await res.text(), ''); }); - // any other store answer says nothing about which store holds the site, so the config read - // decides. keeping the fast source there would answer 503 to a session that cannot recover by - // retrying, since the token is the thing that needs replacing - it('lets the config read answer when the fast store neither served nor refused', async () => { - const { daSourceGet, env, seen } = await build({ - source: DENIED_SOURCE, - fast: FAST, - bus: () => new Response('', { status: 500 }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + // 404 is the only answer a wrong yes produces, so anything else is about the store rather than + // about which store, and refetching it from the config read would hit the same url twice + [429, 500, 502].forEach((status) => { + it(`keeps the fast answer on a store ${status} rather than fetching again`, async () => { + const { daSourceGet, env, seen } = await build({ + source: LEGACY_SOURCE, + fast: FAST, + bus: () => new Response('', { status }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual(res.status, 401); - assert.match(await res.text(), /content="da:401"/); - assert.strictEqual(seen.config, 1); + assert.strictEqual(res.status, status); + assert.strictEqual(seen.bus.length, 1); + assert.strictEqual(seen.legacy.length, 0); + }); }); }); @@ -420,8 +421,6 @@ describe('reading with the content source resolved', () => { }); describe('when the store cannot be reached at all', () => { - // withCorsHeaders reads response.headers, so a throw escaping a handler is an opaque 500 - // with no CORS headers on it it('answers 503 rather than throwing on an html read', async () => { const { daSourceGet, env } = await build({ source: BUS_SOURCE, @@ -536,6 +535,7 @@ describe('reading with the content source resolved', () => { const { daSourceGet } = await esmock('../../src/routes/da-admin.js', { '../../src/storage/content-source.js': { default: async () => BUS_SOURCE, + fastSourceBus: async () => undefined, SOURCE_BUS: 'sourcebus', LEGACY: 'legacy', UNKNOWN: 'unknown', @@ -556,13 +556,13 @@ describe('reading with the content source resolved', () => { }); describe('a non-html read', () => { - it('goes to the source bus with the case it stored the file under', async () => { + it('goes to the source bus fully normalized', async () => { const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Media/holiday.PNG'); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/media/holiday.png'); }); it('goes to da-admin fully lowercased on a legacy site', async () => { diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 0770b214..59b3dbb3 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -253,16 +253,16 @@ describe('writing to the store that holds the site', () => { }); describe('the path a save is written to', () => { - it('keeps the case the source bus stores it under', async () => { + it('is stored under the normalized path on the source bus', async () => { const { seen } = await post( { source: BUS_SOURCE }, 'https://main--site--org.ue.da.live/Folder/Content', ); - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/Folder/content.html'); + assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); }); - it('lowercases the whole path for da-admin', async () => { + it('normalizes the whole path for da-admin', async () => { const { seen } = await post( { source: LEGACY_SOURCE }, 'https://main--site--org.ue.da.live/Folder/Content', diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js index c48721e2..be02e724 100644 --- a/test/storage/content-source.test.js +++ b/test/storage/content-source.test.js @@ -277,24 +277,6 @@ describe('resolveContentSource', () => { }); describe('the API host', () => { - // the caller turns unknown into a 503 it can return; a throw here escapes into - // withCorsHeaders, which reads response.headers and throws again on undefined - it('answers unknown rather than throwing when it is not set', async () => { - stubFetch(legacyBody); - - const source = await resolveContentSource({}, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown rather than throwing when it is not a url', async () => { - stubFetch(legacyBody); - - const source = await resolveContentSource({ AEM_API: 'not-a-url' }, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - it('comes from env, so stage can point elsewhere', async () => { stubFetch(legacyBody); @@ -418,29 +400,5 @@ describe('fastSourceBus', () => { assert.strictEqual(await fastSourceBus(env, daCtx({ site: undefined })), undefined); assert.strictEqual(calls.length, 0); }); - - it('answers undefined when the admin host is unusable', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - assert.strictEqual(await fastSourceBus({ ...env, HLX_ADMIN: 'nope' }, daCtx()), undefined); - }); - - // getStore builds the store url from this base and its `new URL` is unguarded, so a base that - // cannot parse throws out of `worker.fetch` before withCorsHeaders runs: an opaque 500 with no - // CORS on a document read, and a 404 on the raced image path where allSettled swallows it. The - // config read answers unknown for the same env, so the caller gets a 503 instead. - [ - ['AEM_API is unset', undefined], - ['AEM_API has no scheme', 'api.aem.live'], - ['AEM_API is protocol-relative', '//api.aem.live'], - ['AEM_API has a trailing space', 'https://api.aem.live '], - ].forEach(([what, AEM_API]) => { - it(`answers undefined without asking when ${what}`, async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - assert.strictEqual(await fastSourceBus({ ...env, AEM_API }, daCtx()), undefined); - assert.strictEqual(calls.length, 0); - }); - }); }); }); diff --git a/test/storage/store.test.js b/test/storage/store.test.js index 2807843b..b1c2835b 100644 --- a/test/storage/store.test.js +++ b/test/storage/store.test.js @@ -15,46 +15,21 @@ import assert from 'assert'; import { getDaCtx } from '../../src/utils/daCtx.js'; import { LEGACY, SOURCE_BUS } from '../../src/storage/content-source.js'; -const { default: getStore, sourceBusPath } = await import('../../src/storage/store.js'); +const { default: getStore } = await import('../../src/storage/store.js'); const env = { DA_ADMIN: 'https://admin.da.live' }; const ctxFor = (url) => getDaCtx(new Request(url, { headers: { Authorization: 'Bearer t' } })); const legacy = { kind: LEGACY }; const bus = { kind: SOURCE_BUS, base: 'https://api.aem.live/org/sites/site/source' }; -describe('sourceBusPath', () => { - // helix-api-service sanitizes only the basename: computePaths pops the filename, runs - // sanitizeName on it and recombines the directory segments untouched. Verified live on - // 2026-08-03 by uploading /Media/CaseProbe.PNG and fetching six spellings back: only - // /Media/CaseProbe.PNG and /Media/caseprobe.PNG answered 200. - const cases = [ - ['/folder/content', '/folder/content.html', 'appends .html when the request had no extension'], - ['/', '/index.html', 'names the root document index.html'], - ['/sub-folder/', '/sub-folder/index.html', 'names a directory index'], - ['/Media/Holiday.PNG', '/Media/holiday.PNG', 'lowercases the stem, keeps directory and extension case'], - ['/A/B/c.JSON', '/A/B/c.JSON', 'keeps every directory segment as requested'], - ['/Sub-Folder/', '/Sub-Folder/index.html', 'keeps directory case on a directory index'], - ['/folder/Content', '/folder/content.html', 'lowercases a stem that had no extension'], - ['/folder/content.plain.html', '/folder/content.plain.html', 'treats only the last dot as the extension'], - ['/2026.q1/report', '/2026.q1/report.html', 'ignores a dot in a directory segment'], - ]; - - cases.forEach(([path, expected, what]) => { - it(what, () => { - assert.strictEqual(sourceBusPath(ctxFor(`https://main--site--org.ue.da.live${path}`)), expected); - }); - }); - - it('differs from daCtx.sourcePath, which da-admin wants lowercased throughout', () => { - const daCtx = ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'); - - assert.strictEqual(daCtx.sourcePath, '/media/holiday.png'); - assert.strictEqual(sourceBusPath(daCtx), '/Media/holiday.PNG'); - }); -}); - describe('getStore', () => { describe('the url it reads and writes', () => { + it('builds the source-bus url from the normalized path, not the requested one', () => { + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'), bus); + + assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/media/holiday.png'); + }); + it('builds a legacy url under DA_ADMIN from the lowercased source path', () => { const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Folder/Doc'), legacy); @@ -67,12 +42,6 @@ describe('getStore', () => { assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/folder/doc.html'); }); - it('keeps the case the source bus stored a file under', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'), bus); - - assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/Media/holiday.PNG'); - }); - it('takes the base verbatim, so a config naming another org is followed', () => { const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), { kind: SOURCE_BUS, @@ -115,63 +84,77 @@ describe('getStore', () => { }); describe('the write body each store parses', () => { + // both stores are captured with the same recorder, so a test names the store it expects by + // passing its kind rather than by picking a transport + const written = async (kind, html = '') => { + let sent; + const capture = async (input, init) => { + sent = input instanceof Request ? input : new Request(input, init); + return new Response(''); + }; + globalThis.fetch = capture; + const bound = { ...env, daadmin: { fetch: capture } }; + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), kind); + + await store.write(html, 'Bearer t'); + + delete globalThis.fetch; + return sent; + }; + // helix-api-service parses no form data anywhere: getValidPayload reads the raw buffer and // types it from the path extension. Sending da-admin's multipart envelope stores the // boundary lines as the document text and answers 201, so this is not a cosmetic difference. it('sends the source bus the document as the raw body', async () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); - - const init = store.writeInit('

hi

', 'Bearer t'); - const body = await new Request('https://example.test', { method: 'POST', ...init }).text(); + const sent = await written(bus, '

hi

'); - assert.strictEqual(body, '

hi

'); + assert.strictEqual(await sent.text(), '

hi

'); }); - it('types the source-bus write as text/html', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); - - const init = store.writeInit('', 'Bearer t'); + it('types the source-bus write as text/html', async () => { + const sent = await written(bus); - assert.strictEqual(new Headers(init.headers).get('Content-Type'), 'text/html'); + assert.strictEqual(sent.headers.get('Content-Type'), 'text/html'); }); it('sends da-admin the document as a data form part', async () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + const sent = await written(legacy, '

hi

'); - const init = store.writeInit('

hi

', 'Bearer t'); - const form = await new Request('https://example.test', { method: 'POST', ...init }).formData(); + const form = await sent.formData(); assert.strictEqual(await form.get('data').text(), '

hi

'); assert.strictEqual(form.get('data').type, 'text/html'); }); it('never wraps a source-bus write in a multipart envelope', async () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); - - const init = store.writeInit('', 'Bearer t'); - const body = await new Request('https://example.test', { method: 'POST', ...init }).text(); + const body = await (await written(bus)).text(); assert.ok(!body.includes('Content-Disposition'), `envelope leaked into the body: ${body}`); assert.ok(!body.includes('form-data'), `envelope leaked into the body: ${body}`); }); - it('authorizes both writes with the caller token', () => { - const b = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); - const l = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + it('posts to the url the store resolved', async () => { + const b = await written(bus); + const l = await written(legacy); + + assert.strictEqual(b.method, 'POST'); + assert.strictEqual(b.url, 'https://api.aem.live/org/sites/site/source/doc.html'); + assert.strictEqual(l.method, 'POST'); + assert.strictEqual(l.url, 'https://admin.da.live/source/org/site/doc.html'); + }); - assert.strictEqual(new Headers(b.writeInit('', 'Bearer t').headers).get('Authorization'), 'Bearer t'); - assert.strictEqual(new Headers(l.writeInit('', 'Bearer t').headers).get('Authorization'), 'Bearer t'); + it('authorizes both writes with the caller token', async () => { + assert.strictEqual((await written(bus)).headers.get('Authorization'), 'Bearer t'); + assert.strictEqual((await written(legacy)).headers.get('Authorization'), 'Bearer t'); }); // neither store's writes are conditional. Only the source bus sets an etag on a read, and // nothing carries it into the save: a marker on the connection uri would be minted once per // page load while UE saves many times against it, so no version pin could stay fresh. - it('sends no precondition to either store', () => { - const b = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), bus); - const l = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + it('sends no precondition to either store', async () => { + const sent = [await written(bus), await written(legacy)]; - [b, l].forEach((store) => { - const headers = new Headers(store.writeInit('', 'Bearer t').headers); + sent.forEach(({ headers }) => { assert.strictEqual(headers.get('If-Match'), null); assert.strictEqual(headers.get('If-None-Match'), null); assert.strictEqual(headers.get('If-Unmodified-Since'), null); diff --git a/test/utils/daCtx.test.js b/test/utils/daCtx.test.js index 96a4a9ca..fd57c7ee 100644 --- a/test/utils/daCtx.test.js +++ b/test/utils/daCtx.test.js @@ -193,6 +193,18 @@ describe('DA context', () => { assert.strictEqual(ctx.ext, 'png'); }); + it('maps /A/B/c.JSON to /a/b/c.json', () => { + const ctx = ctxFor('/A/B/c.JSON'); + assert.strictEqual(ctx.sourcePath, '/a/b/c.json'); + assert.strictEqual(ctx.ext, 'json'); + }); + + it('maps /folder/Sub-Folder/ to /folder/sub-folder/index.html', () => { + const ctx = ctxFor('/folder/Sub-Folder/'); + assert.strictEqual(ctx.sourcePath, '/folder/sub-folder/index.html'); + assert.strictEqual(ctx.ext, 'html'); + }); + it('maps /sheet.json to /sheet.json', () => { const ctx = ctxFor('/sheet.json'); assert.strictEqual(ctx.sourcePath, '/sheet.json'); From 2f6b18b23bd13625055d6ae95faf2e61174766c8 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Thu, 6 Aug 2026 22:45:41 +0200 Subject: [PATCH 42/48] test: cover the worker.fetch backstop, 500 with CORS on a throwing handler --- test/index.test.js | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/test/index.test.js b/test/index.test.js index 133e1b5e..7044541d 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -32,7 +32,62 @@ const HANDLER_MOCKS = { }, }; +const throwingWorker = async (handler) => (await esmock('../src/index.js', { + ...HANDLER_MOCKS, + [handler]: { + default: async () => { + throw new TypeError('fetch failed'); + }, + }, +})).default; + describe('worker fetch handler', () => { + describe('when a handler throws', () => { + // a throw used to reject worker.fetch, and the runtime answered a bare 500 with no CORS on it, + // which the editor cannot read at all + it('answers 500 rather than rejecting', async () => { + const worker = await throwingWorker('../src/handlers/get.js'); + const req = new Request('https://main--site--org.ue.da.live/some/path'); + + const res = await worker.fetch(req, {}); + + assert.ok(res instanceof Response, 'must return a Response'); + assert.strictEqual(res.status, 500); + }); + + // the assertion that pins the catch inside the switch: wrapped around withCorsHeaders instead, + // the status would still be 500 and the headers would be gone + it('keeps the CORS headers on it', async () => { + const worker = await throwingWorker('../src/handlers/get.js'); + const req = new Request('https://main--site--org.ue.da.live/some/path', { + headers: { Origin: 'https://da.live' }, + }); + + const res = await worker.fetch(req, {}); + + assert.strictEqual(res.headers.get('Access-Control-Allow-Origin'), 'https://da.live'); + assert.strictEqual(res.headers.get('Access-Control-Allow-Credentials'), 'true'); + }); + + it('answers the same on a POST', async () => { + const worker = await throwingWorker('../src/handlers/post.js'); + const req = new Request('https://main--site--org.ue.da.live/some/path', { method: 'POST' }); + + const res = await worker.fetch(req, {}); + + assert.strictEqual(res.status, 500); + }); + + it('sends no body, so nothing internal reaches the browser', async () => { + const worker = await throwingWorker('../src/handlers/get.js'); + const req = new Request('https://main--site--org.ue.da.live/some/path'); + + const res = await worker.fetch(req, {}); + + assert.strictEqual(await res.text(), ''); + }); + }); + describe('/.rum/ routing', () => { let worker; From 0c202d5f524ddb74303321863cd44a127543d7dd Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Fri, 7 Aug 2026 09:24:27 +0200 Subject: [PATCH 43/48] fix: decide the store on /ping alone, refuse source-bus writes the sidekick read is gone, so is the unknown kind and its 503. `/ping` and the presence of x-api-upgrade-available is the whole test, the same one da-nx applies in isHlx6. a write to a source-bus site answers 405 and touches no store, since UE on da.live never supported them. --- src/handlers/post.js | 3 + src/responses/index.js | 10 + src/routes/da-admin.js | 98 ++---- src/storage/content-source.js | 127 -------- src/storage/source-bus.js | 36 +++ src/storage/store.js | 37 +-- src/utils/constants.js | 4 +- test/handlers/post.test.js | 54 ++++ test/index.test.js | 51 ++- test/routes/da-admin.test.js | 81 +++-- test/routes/source-read.test.js | 485 +++++++++------------------- test/routes/source-write.test.js | 248 +++++++------- test/storage/content-source.test.js | 404 ----------------------- test/storage/source-bus.test.js | 179 ++++++++++ test/storage/store.test.js | 119 ++----- 15 files changed, 708 insertions(+), 1228 deletions(-) delete mode 100644 src/storage/content-source.js create mode 100644 src/storage/source-bus.js create mode 100644 test/handlers/post.test.js delete mode 100644 test/storage/content-source.test.js create mode 100644 test/storage/source-bus.test.js diff --git a/src/handlers/post.js b/src/handlers/post.js index ca12c050..c32bc5c8 100644 --- a/src/handlers/post.js +++ b/src/handlers/post.js @@ -9,9 +9,12 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ +import { get404 } from '../responses/index.js'; import { daSourcePost } from '../routes/da-admin.js'; export default async function postHandler({ req, env, daCtx }) { + if (!daCtx.site) return get404(); + // for now forward all POST requests to the da-admin return daSourcePost({ req, env, daCtx }); } diff --git a/src/responses/index.js b/src/responses/index.js index e1639e8d..499a5a52 100644 --- a/src/responses/index.js +++ b/src/responses/index.js @@ -69,6 +69,16 @@ export function post503(message = '') { }); } +// RFC 9110 requires an Allow header on a 405, and reads are what is left once the write is gone. +export function post405(message = '') { + return daResp({ + body: message, + status: 405, + contentType: 'text/plain; charset=utf-8', + headers: [['Allow', 'GET, HEAD, OPTIONS']], + }); +} + export function head401() { return new Response(null, { status: 401 }); } diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index c628ec04..a8f8566b 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -22,19 +22,18 @@ import { applyQuickEditToDocument, buildQuickEditCookie, buildQuickEditNotFoundResponse, } from '../utils/quick-edit.js'; import { - daResp, get401, get404, get415, get503, head401, head503, post503, + daResp, get401, get404, get415, get503, head401, head503, post405, post503, } from '../responses/index.js'; import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, + SOURCE_BUS_READ_ONLY_MESSAGE, SOURCE_UNREACHABLE_HTML_MESSAGE, SOURCE_UNREACHABLE_MESSAGE, - SOURCE_UNRESOLVED_HTML_MESSAGE, - SOURCE_UNRESOLVED_MESSAGE, UNAUTHORIZED_HTML_MESSAGE, } from '../utils/constants.js'; import { getSiteConfig } from '../storage/config.js'; -import resolveContentSource, { fastSourceBus, UNAUTHORIZED, UNKNOWN } from '../storage/content-source.js'; +import isSourceBus from '../storage/source-bus.js'; import getStore from '../storage/store.js'; import { restoreAbsoluteImages } from '../render/rewrite-images.js'; @@ -99,37 +98,14 @@ async function reachStore(store, send) { } /** - * Reads from the store that holds the site, taking `/ping`'s fast answer when the store replied. + * Reads from the store that holds the site. * - * The two lookups run at once: `/ping` answers an enrolled site from the Fastly edge in ~37ms, - * where the config read is ~529ms, and a legacy site learns nothing from `/ping` so it would - * otherwise pay both in series. - * - * A 404 is the one answer that falls through to the config read, since a wrong yes is what - * produces one and the starter template would then be composed over a document that exists in the - * other store. Any other status is about the store rather than about which store, so returning it - * saves fetching the same url twice. - * - * @returns {Promise<{source: Object, response?: Response}>} `response` is absent when the store - * could not be reached, and when `source.kind` is unauthorized or unknown + * @returns {Promise} undefined when the store could not be reached */ async function readSource(env, daCtx, init) { - const config = resolveContentSource(env, daCtx); - const fast = await fastSourceBus(env, daCtx); - if (fast) { - const store = getStore(env, daCtx, fast); - console.log(`-> ${init.method} ${store.url.toString()} (fast)`); - const response = await reachStore(store, () => store.fetch(store.url, init)); - if (response && response.status !== 404) return { source: fast, response }; - } - - const source = await config; - if (source.kind === UNAUTHORIZED || source.kind === UNKNOWN) return { source }; - - const store = getStore(env, daCtx, source); + const store = getStore(env, daCtx, await isSourceBus(env, daCtx)); console.log(`-> ${init.method} ${store.url.toString()}`); - const response = await reachStore(store, () => store.fetch(store.url, init)); - return { source, response }; + return reachStore(store, () => store.fetch(store.url, init)); } export async function daSourceGet({ req, env, daCtx }) { @@ -154,14 +130,7 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing. A refusal is passed on as // itself: nothing renders an image, so the da:401 shell would only corrupt it. - const { source, response } = await readSource(env, daCtx, { method: 'GET', headers }); - if (source.kind === UNAUTHORIZED) { - return daResp({ body: '', status: source.status, contentType: 'text/plain; charset=utf-8' }); - } - if (source.kind === UNKNOWN) { - console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); - return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); - } + const response = await readSource(env, daCtx, { method: 'GET', headers }); if (!response) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); console.log(`<- ${daCtx.sourcePath}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; @@ -169,11 +138,10 @@ export async function daSourceGet({ req, env, daCtx }) { // the store lookup costs a round trip, so it runs alongside head.html rather than after it const aemCtx = getAemCtx(env, daCtx); - const [headHtml, read] = await Promise.all([ + const [headHtml, sourceResp] = await Promise.all([ getAEMHtml(aemCtx, '/head.html'), readSource(env, daCtx, { method: 'GET', headers }), ]); - const { source, response: sourceResp } = read; if (!headHtml) { // quick-edit still needs a working shell (with the import map) so the editor // can load into this page, even when the AEM branch doesn't exist yet. @@ -182,19 +150,11 @@ export async function daSourceGet({ req, env, daCtx }) { } return get404(BRANCH_NOT_FOUND_HTML_MESSAGE); } - if (source.kind === UNAUTHORIZED) { - return daResp({ body: UNAUTHORIZED_HTML_MESSAGE, status: source.status, contentType: 'text/html' }); - } - if (source.kind === UNKNOWN) { - console.warn(`503 GET ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); - return get503(SOURCE_UNRESOLVED_HTML_MESSAGE); - } if (!sourceResp) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); console.log(`<- ${daCtx.sourcePath}. ${sourceResp.status} ${sourceResp.statusText}`, { status: sourceResp.status, statusText: sourceResp.statusText }); - // the store is the first thing to see the token when the fast path skipped the config read, and - // the authorbus extension recovers off the da:401 meta rather than the status, so a refusal from - // the store gets the same shell the config read would have produced + // the store is the only thing to see the token, and the authorbus extension recovers off the + // da:401 meta rather than the status, so a refusal from the store gets that shell if (sourceResp.status === 401 || sourceResp.status === 403) { return daResp({ body: UNAUTHORIZED_HTML_MESSAGE, status: sourceResp.status, contentType: 'text/html' }); } @@ -247,14 +207,7 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const { source, response } = await readSource(env, daCtx, { method: 'HEAD', headers }); - if (source.kind === UNAUTHORIZED) { - return new Response(null, { status: source.status }); - } - if (source.kind === UNKNOWN) { - console.warn(`503 HEAD ${daCtx.sourcePath}, content source unresolved: ${source.reason}`); - return head503(); - } + const response = await readSource(env, daCtx, { method: 'HEAD', headers }); if (!response) return head503(); console.log(`<- HEAD ${daCtx.sourcePath}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return new Response(null, { status: response.status, headers: response.headers }); @@ -294,21 +247,24 @@ export async function daSourcePost({ req, env, daCtx }) { const bodyContent = toHtml(bodyNode); - // the payload is settled, so the only question left is where it goes. A write is the one - // operation a wrong guess cannot be walked back from. - const source = await resolveContentSource(env, daCtx); - if (source.kind === UNAUTHORIZED) { - return daResp({ body: '', status: source.status, contentType: 'text/plain; charset=utf-8' }); - } - if (source.kind === UNKNOWN) { - console.warn(`503 POST ${sourcePath}, content source unresolved: ${source.reason}`); - return post503(SOURCE_UNRESOLVED_MESSAGE); + // the payload is settled, so the only question left is where it goes + const onSourceBus = await isSourceBus(env, daCtx); + + if (onSourceBus) { + console.log(`405 POST ${sourcePath}, writes to the source bus are refused through the preview proxy. write directly to the source bus instead.`); + return post405(SOURCE_BUS_READ_ONLY_MESSAGE); } - // the two stores take the document in different shapes, so the store sends its own request - const store = getStore(env, daCtx, source); + // da-admin takes the document as a `data` form part + const store = getStore(env, daCtx, onSourceBus); + const body = new FormData(); + body.set('data', new Blob([bodyContent], { type: 'text/html' })); console.log(`-> ${store.url.toString()}`); - const response = await reachStore(store, () => store.write(bodyContent, authToken)); + const response = await reachStore(store, () => store.fetch(new Request(store.url, { + method: 'POST', + body, + headers: { Authorization: authToken }, + }))); if (!response) return post503(SOURCE_UNREACHABLE_MESSAGE); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; diff --git a/src/storage/content-source.js b/src/storage/content-source.js deleted file mode 100644 index cdaaeed3..00000000 --- a/src/storage/content-source.js +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2026 Adobe. All rights reserved. - * This file is licensed to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS - * OF ANY KIND, either express or implied. See the License for the specific language - * governing permissions and limitations under the License. - */ - -const LEGACY_PREFIX = 'https://content.da.live/'; -const TIMEOUT_MS = 5 * 1000; -const UPGRADE_HEADER = 'x-api-upgrade-available'; - -export const SOURCE_BUS = 'sourcebus'; -export const LEGACY = 'legacy'; -export const UNKNOWN = 'unknown'; -export const UNAUTHORIZED = 'unauthorized'; - -/** - * Asks `/ping` whether a site is on the source bus, for a read. - * - * The Fastly edge answers an enrolled site from a dictionary in ~37ms without reaching an origin, - * against ~529ms for the config read. Only the yes is usable: helix-admin sets the header when - * config resolution succeeded and named the API, so its absence covers a legacy site, a config - * that would not resolve, and a site that does not exist alike. - * - * The base is built rather than read, because `/ping` returns a header and no url. - * helix-api-service parses org and site out of a source url and refuses one that names another - * site (`src/contentproxy/source/utils.js`, "only allow source bus from the same org and site"), - * so this is the only base the site can legally have. - * - * @returns {Promise<{kind: string, base: string}|undefined>} undefined whenever `/ping` did not - * say yes, which leaves the config read to answer - */ -export async function fastSourceBus(env, daCtx) { - const { org, site } = daCtx; - if (!org || !site) return undefined; - - const url = new URL(`/ping/${org}/${site}`, env.HLX_ADMIN); - try { - const response = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }); - if (response.status !== 200) return undefined; - if (response.headers.get(UPGRADE_HEADER) !== 'true') return undefined; - } catch (e) { - console.warn(`[source] ${org}/${site} ping failed: ${e.name}: ${e.message}`); - return undefined; - } - - return { kind: SOURCE_BUS, base: `${env.AEM_API}/${org}/sites/${site}/source` }; -} - -function unknown(org, site, reason) { - console.warn(`[source] ${org}/${site} unknown: ${reason}`); - return { kind: UNKNOWN, reason }; -} - -/** - * Asks the AEM API which store holds a site's content. - * - * `GET {AEM_API}/{org}/sites/{site}/sidekick` returns the resolved content source url in its body - * and needs only `code:read`, the permission every authoring role already has. It answers for - * legacy sites too, because both stores read the same config service. A config that could not be - * resolved is a 404 rather than a wrong answer, which is what lets an unresolved source be - * refused instead of guessed at. - * - * The prefix test is the same one the platform applies to itself: - * `helix-api-service/src/contentproxy/index.js` reads the source as the source bus when its url - * starts with the API host. - * - * @param {Object} env worker env, `AEM_API` is the API host and the source-bus prefix - * @param {Object} daCtx - * @returns {Promise<{kind: string, base?: string, status?: number, reason?: string}>} `sourcebus` - * with the store base url, `legacy`, `unauthorized` with the status the API gave, or `unknown` - * with the reason it could not be answered - */ -export default async function resolveContentSource(env, daCtx) { - const { org, site, authToken } = daCtx; - - // an unparseable hostname leaves org and site undefined, and there is no site to ask about - if (!org || !site) { - return unknown(org, site, 'no org or site in the request'); - } - - const url = new URL(`/${org}/sites/${site}/sidekick`, env.AEM_API); - const headers = new Headers(); - if (authToken) headers.set('Authorization', authToken); - - let response; - try { - response = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS) }); - } catch (e) { - return unknown(org, site, `${url} failed with ${e.name}: ${e.message}`); - } - - // an expired or insufficient session is a definite answer, not an unresolved one. Reporting it - // as unresolved answers a retryable 503, and the caller re-tries a session that cannot recover. - if (response.status === 401 || response.status === 403) { - console.warn(`[source] ${org}/${site} not authorized: ${url} answered ${response.status}`); - return { kind: UNAUTHORIZED, status: response.status }; - } - - if (response.status !== 200) { - return unknown(org, site, `${url} answered ${response.status}`); - } - - let config; - try { - config = await response.json(); - } catch (e) { - return unknown(org, site, `${url} did not answer json: ${e.message}`); - } - - const sourceUrl = config?.contentSourceUrl; - if (typeof sourceUrl !== 'string') { - return unknown(org, site, `${url} named no content source`); - } - if (sourceUrl.startsWith(`${env.AEM_API}/`)) { - return { kind: SOURCE_BUS, base: sourceUrl.replace(/\/$/, '') }; - } - if (sourceUrl.startsWith(LEGACY_PREFIX)) { - return { kind: LEGACY }; - } - return unknown(org, site, `content source ${sourceUrl} is neither store`); -} diff --git a/src/storage/source-bus.js b/src/storage/source-bus.js new file mode 100644 index 00000000..c74c394c --- /dev/null +++ b/src/storage/source-bus.js @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +const TIMEOUT_MS = 5 * 1000; +const UPGRADE_HEADER = 'x-api-upgrade-available'; + +/** + * Asks `/ping` whether a site is on the source bus. + * @param {Object} env worker env, `HLX_ADMIN` is where the probe goes + * @param {Object} daCtx + * @returns {Promise} + */ +export default async function isSourceBus(env, daCtx) { + const { org, site } = daCtx; + // an unparseable hostname leaves org and site undefined, and there is no site to ask about + if (!org || !site) return false; + + const key = `${org}/${site}`; + try { + const url = new URL(`/ping/${key}`, env.HLX_ADMIN); + const response = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }); + return response.headers.get(UPGRADE_HEADER) !== null; + } catch (e) { + console.warn(`[source] ${key} ping failed: ${e.name}: ${e.message}`); + return false; + } +} diff --git a/src/storage/store.js b/src/storage/store.js index 95cc359f..e3873de6 100644 --- a/src/storage/store.js +++ b/src/storage/store.js @@ -9,50 +9,29 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ -import { SOURCE_BUS } from './content-source.js'; - /** - * Picks the store for a request, the way to reach it, and the shape it takes a write in. + * Picks the store that holds a site's content, the url the document has in it, and the way to + * reach it. * * da-admin answers over a service binding and the source bus over the public network, so one - * fetch cannot serve both. They also differ on the write body: helix-api-service reads the raw - * request body and types it from the path extension, parsing no form data anywhere, while - * da-admin takes the document as a `data` form part. Handing either the other's shape stores - * something other than the document and answers 201. `write` sends the document so the caller - * never assembles either shape. + * fetch cannot serve both and the caller cannot pick the transport for itself. * * @param {Object} env worker env * @param {Object} daCtx - * @param {{kind: string, base?: string}} source the resolved content source + * @param {boolean} onSourceBus whether the site is enrolled on the source bus */ -export default function getStore(env, daCtx, source) { +export default function getStore(env, daCtx, onSourceBus) { const { org, site, sourcePath } = daCtx; - if (source.kind === SOURCE_BUS) { - const url = new URL(`${source.base}${sourcePath}`); + if (onSourceBus) { return { - url, + url: new URL(`/${org}/sites/${site}/source${sourcePath}`, env.AEM_API), fetch: (input, init) => fetch(input, init), - write: (html, authToken) => fetch(new Request(url, { - method: 'POST', - body: html, - headers: { Authorization: authToken, 'Content-Type': 'text/html' }, - })), }; } - const url = new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN); return { - url, + url: new URL(`/source/${org}/${site}${sourcePath}`, env.DA_ADMIN), fetch: (input, init) => env.daadmin.fetch(input, init), - write: (html, authToken) => { - const body = new FormData(); - body.set('data', new Blob([html], { type: 'text/html' })); - return env.daadmin.fetch(new Request(url, { - method: 'POST', - body, - headers: { Authorization: authToken }, - })); - }, }; } diff --git a/src/utils/constants.js b/src/utils/constants.js index b6abe7cb..b26a5f5c 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -51,12 +51,10 @@ export const DEFAULT_HTML_TEMPLATE = '

Not found: Unable to retrieve AEM branch

'; -export const SOURCE_UNRESOLVED_HTML_MESSAGE = '

503: Content source unresolved

The store that holds this document could not be determined. Please retry.

'; - export const SOURCE_UNREACHABLE_HTML_MESSAGE = '

503: Content store unreachable

The store that holds this document did not answer. Please retry.

'; export const SOURCE_UNREACHABLE_MESSAGE = 'The store that holds this document did not answer, so nothing was written. Please retry.'; -export const SOURCE_UNRESOLVED_MESSAGE = 'The store that holds this document could not be determined, so the write was refused rather than sent to the wrong one. Please retry.'; +export const SOURCE_BUS_READ_ONLY_MESSAGE = 'This site is on the source bus, which this proxy only reads. Nothing was written, and retrying will not help.'; export const DEFAULT_UNAUTHORIZED_HTML_MESSAGE = '

401: Unauthorized

'; diff --git a/test/handlers/post.test.js b/test/handlers/post.test.js new file mode 100644 index 00000000..9c532796 --- /dev/null +++ b/test/handlers/post.test.js @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import esmock from 'esmock'; + +const { getDaCtx } = await import('../../src/utils/daCtx.js'); + +describe('POST handler', () => { + let postHandler; + let writes; + + beforeEach(async () => { + writes = 0; + postHandler = (await esmock('../../src/handlers/post.js', { + '../../src/routes/da-admin.js': { + daSourcePost: async () => { + writes += 1; + return new Response('', { status: 201 }); + }, + }, + })).default; + }); + + // the store url is built from org and site, so a hostname naming neither would send the author + // token to https://admin.da.live/source/undefined/undefined/... + it('answers 404 when the hostname named no site, and writes nothing', async () => { + const req = new Request('https://xyz.ue.da.live/content', { method: 'POST' }); + + const res = await postHandler({ req, env: {}, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 404); + assert.strictEqual(writes, 0); + }); + + it('forwards a write on a site it can name', async () => { + const req = new Request('https://main--site--org.ue.da.live/folder/content', { method: 'POST' }); + + const res = await postHandler({ req, env: {}, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 201); + assert.strictEqual(writes, 1); + }); +}); diff --git a/test/index.test.js b/test/index.test.js index 7044541d..0bdacba0 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -14,13 +14,11 @@ import assert from 'assert'; import esmock from 'esmock'; -const HANDLER_MOCKS = { +// everything but the POST handler, so a write can be driven through the real route +const READ_HANDLER_MOCKS = { '../src/handlers/get.js': { default: async () => new Response('get-handled', { status: 200 }), }, - '../src/handlers/post.js': { - default: async () => new Response('post-handled', { status: 200 }), - }, '../src/handlers/options.js': { default: async () => new Response('options-handled', { status: 204 }), }, @@ -32,6 +30,13 @@ const HANDLER_MOCKS = { }, }; +const HANDLER_MOCKS = { + ...READ_HANDLER_MOCKS, + '../src/handlers/post.js': { + default: async () => new Response('post-handled', { status: 200 }), + }, +}; + const throwingWorker = async (handler) => (await esmock('../src/index.js', { ...HANDLER_MOCKS, [handler]: { @@ -179,4 +184,42 @@ describe('worker fetch handler', () => { assert.strictEqual(res.headers.get('Access-Control-Allow-Headers'), 'Authorization, Content-Type, x-site-token'); }); }); + + // withCorsHeaders rebuilds every response from the handler, so a header the route set only + // reaches the caller if that rebuild carries it + describe('a refused write on a source-bus site', () => { + const busWorker = async () => (await esmock('../src/index.js', READ_HANDLER_MOCKS, { + '../src/storage/source-bus.js': { default: async () => true }, + })).default; + + const uePost = (origin) => { + const body = new FormData(); + body.set('data', new File(['

x

'], 'c.html', { type: 'text/html' })); + return new Request('https://main--site--org.ue.da.live/folder/content', { + method: 'POST', + body, + headers: { Authorization: 'Bearer t', Origin: origin }, + }); + }; + + it('keeps the Allow header alongside the CORS headers', async () => { + const worker = await busWorker(); + + const res = await worker.fetch(uePost('https://da.live'), {}); + + assert.strictEqual(res.status, 405); + assert.strictEqual(res.headers.get('Allow'), 'GET, HEAD, OPTIONS'); + assert.strictEqual(res.headers.get('Access-Control-Allow-Origin'), 'https://da.live'); + assert.strictEqual(res.headers.get('Access-Control-Allow-Credentials'), 'true'); + }); + + it('keeps it for an untrusted origin too', async () => { + const worker = await busWorker(); + + const res = await worker.fetch(uePost('https://evil.example.com'), {}); + + assert.strictEqual(res.headers.get('Allow'), 'GET, HEAD, OPTIONS'); + assert.strictEqual(res.headers.get('Access-Control-Allow-Origin'), '*'); + }); + }); }); diff --git a/test/routes/da-admin.test.js b/test/routes/da-admin.test.js index cef2dbb6..aad41fc2 100644 --- a/test/routes/da-admin.test.js +++ b/test/routes/da-admin.test.js @@ -44,12 +44,23 @@ const recorder = () => { return { env, fetched }; }; +// answers the real /ping, which is the only lookup the routes make. `upgraded` is the set of +// `org/site` keys the probe reports as enrolled. +const stubPing = (upgraded = []) => { + const asked = []; + globalThis.fetch = async (input) => { + const url = input.toString(); + asked.push(url); + const key = url.slice(url.indexOf('/ping/') + '/ping/'.length); + const headers = upgraded.includes(key) ? { 'x-api-upgrade-available': 'true' } : {}; + return new Response('', { status: 200, headers }); + }; + return asked; +}; + const mockRoutes = async () => esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { - default: async () => ({ kind: 'legacy' }), - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', + '../../src/storage/source-bus.js': { + default: async () => false, }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({}), @@ -102,12 +113,8 @@ describe('daSourceGet', () => { const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; calls = { compose: [], ue: 0, quickEdit: 0 }; return (await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { - default: async () => ({ kind: 'legacy' }), - fastSourceBus: async () => undefined, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', + '../../src/storage/source-bus.js': { + default: async () => false, }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({}), @@ -257,13 +264,10 @@ describe('daSourceGet', () => { }); describe('source URLs', () => { - // these drive the unmocked module, so the content-source lookup really does reach out; - // answer it as the legacy store, which is the store these tests describe + // these drive the unmocked module, so the /ping lookup really does reach out; answer it + // without the upgrade header, which is the legacy store these tests describe beforeEach(() => { - globalThis.fetch = async () => new Response( - JSON.stringify({ contentSourceUrl: 'https://content.da.live/org/site/' }), - { status: 200 }, - ); + stubPing(); }); afterEach(() => { @@ -398,19 +402,50 @@ describe('daSourcePost to a non-HTML path', () => { }); describe('daSourcePost', () => { - // these drive the unmocked module, so the content-source lookup really does reach out; - // answer it as the legacy store, which is the store these tests describe + // these drive the unmocked module, so the /ping lookup really does reach out; answer it + // without the upgrade header, which is the legacy store these tests describe beforeEach(() => { - globalThis.fetch = async () => new Response( - JSON.stringify({ contentSourceUrl: 'https://content.da.live/org/site/' }), - { status: 200 }, - ); + stubPing(); }); afterEach(() => { delete globalThis.fetch; }); + describe('on a site /ping reports as enrolled', () => { + const write = async (site, env) => { + const html = new File(['hello'], 'page.html', { type: 'text/html' }); + const req = formReq(`https://main--${site}--org.ue.da.live/page`, html); + return daSourcePost({ req, env, daCtx: getDaCtx(req) }); + }; + + it('is refused with 405 and nothing is written', async () => { + stubPing(['org/refused']); + const { env, fetched } = recorder(); + + const res = await write('refused', env); + + assert.strictEqual(res.status, 405); + assert.strictEqual(res.headers.get('Allow'), 'GET, HEAD, OPTIONS'); + assert.deepStrictEqual(fetched, []); + }); + + // nothing is remembered between requests, so a site enrolled or un-enrolled mid-session takes + // effect on the next one + it('probes once per write', async () => { + const asked = stubPing(['org/probedeach']); + const { env } = recorder(); + + await write('probedeach', env); + await write('probedeach', env); + + assert.deepStrictEqual(asked, [ + 'https://admin.hlx.page/ping/org/probedeach', + 'https://admin.hlx.page/ping/org/probedeach', + ]); + }); + }); + // on an HTML path, so the path check does not answer first and this exercises // the part-type check it('refuses a binary File with 415 and does not write', async () => { diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index a5950e4f..5af043e7 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -15,12 +15,6 @@ import assert from 'assert'; import esmock from 'esmock'; import { getDaCtx } from '../../src/utils/daCtx.js'; -const LEGACY_SOURCE = { kind: 'legacy' }; -const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; -const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the config service answered 503' }; -const DENIED_SOURCE = { kind: 'unauthorized', status: 401 }; -const FORBIDDEN_SOURCE = { kind: 'unauthorized', status: 403 }; - const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer t' } }); /** @@ -30,7 +24,7 @@ const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer */ const build = async (overrides = {}) => { const { - source = LEGACY_SOURCE, + onSourceBus = false, bus = () => new Response('from the source bus', { status: 200, headers: { etag: '"busetag"' } }), legacy = () => new Response('from da-admin', { status: 200 }), } = overrides; @@ -38,7 +32,7 @@ const build = async (overrides = {}) => { // `{ headHtml: undefined }` really does simulate a missing head.html const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; const seen = { - bus: [], legacy: [], ue: 0, ping: 0, config: 0, + bus: [], legacy: [], ue: 0, lookups: 0, }; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); @@ -57,19 +51,11 @@ const build = async (overrides = {}) => { }, }; const mod = await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { + '../../src/storage/source-bus.js': { default: async () => { - seen.config += 1; - return source; - }, - fastSourceBus: async () => { - seen.ping += 1; - return overrides.fast; + seen.lookups += 1; + return onSourceBus; }, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', - UNAUTHORIZED: 'unauthorized', }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({}), @@ -89,163 +75,14 @@ const build = async (overrides = {}) => { return { ...mod, env, seen }; }; -afterEach(() => { - delete globalThis.fetch; -}); - -describe('the /ping fast path on a read', () => { - const FAST = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; - - it('serves the source-bus document without waiting on the config read', async () => { - const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 200); - assert.strictEqual(seen.ping, 1); - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); - }); - - it('takes it on a non-html read too, which is the per-image path', async () => { - const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); - const req = authedReq('https://main--site--org.ue.da.live/media/holiday.png'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 200); - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/media/holiday.png'); - }); - - it('takes it on a HEAD', async () => { - const { daSourceHead, env, seen } = await build({ source: UNKNOWN_SOURCE, fast: FAST }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 200); - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.bus[0].method, 'HEAD'); - }); - - // /ping is trusted only when it produced content. its absence conflates legacy with a config - // that would not resolve, and the CF port of the edge hardcodes the yes, so a yes that finds - // nothing must not become the starter template for the author to save over - it('falls back to the config read when the fast store has nothing', async () => { - const { daSourceGet, env, seen } = await build({ - source: LEGACY_SOURCE, - fast: FAST, - bus: () => new Response('', { status: 404 }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 200); - assert.strictEqual(await res.text(), 'from da-admin'); - assert.strictEqual(seen.legacy.length, 1); - }); - - it('falls back when the fast store cannot be reached', async () => { - const { daSourceGet, env, seen } = await build({ - source: LEGACY_SOURCE, - fast: FAST, - bus: () => { - throw new TypeError('fetch failed'); - }, - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(seen.legacy.length, 1); - }); - - it('does not take it when /ping did not say yes', async () => { - const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE, fast: undefined }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(seen.ping, 1); - assert.strictEqual(seen.bus.length, 0); - assert.strictEqual(seen.legacy.length, 1); - }); - - // a save is one request where 460ms does not matter, and a wrong store on a write cannot be - // walked back - it('is never taken on a write', async () => { - const { daSourcePost, env, seen } = await build({ source: LEGACY_SOURCE, fast: FAST }); - const body = new FormData(); - body.set('data', new File(['

x

'], 'c.html', { type: 'text/html' })); - const req = new Request('https://main--site--org.ue.da.live/folder/content', { - method: 'POST', body, headers: { Authorization: 'Bearer t' }, - }); - - await daSourcePost({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(seen.ping, 0); - assert.strictEqual(seen.bus.length, 0); - assert.strictEqual(seen.legacy.length, 1); - }); - - // /ping reads no token, so on the fast path the store is the first thing to see one. the - // authorbus extension recovers off the da:401 meta and never reads the status, and both stores - // answer 401 with an empty body. the config answer is legacy in these two, so the 401 can only - // have come from the fast store: with a denied config answer they pass either way. - it('serves the da:401 shell when the store refuses the token on an html read', async () => { - const { daSourceGet, env } = await build({ - source: LEGACY_SOURCE, - fast: FAST, - bus: () => new Response('', { status: 401 }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 401); - assert.match(await res.text(), /content="da:401"/); - }); - - it('passes a store 401 through bare on a non-html read, which renders nothing', async () => { - const { daSourceGet, env } = await build({ - source: LEGACY_SOURCE, - fast: FAST, - bus: () => new Response('', { status: 401 }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 401); - assert.strictEqual(await res.text(), ''); - }); - - // 404 is the only answer a wrong yes produces, so anything else is about the store rather than - // about which store, and refetching it from the config read would hit the same url twice - [429, 500, 502].forEach((status) => { - it(`keeps the fast answer on a store ${status} rather than fetching again`, async () => { - const { daSourceGet, env, seen } = await build({ - source: LEGACY_SOURCE, - fast: FAST, - bus: () => new Response('', { status }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, status); - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.legacy.length, 0); - }); +describe('reading from the store that holds the site', () => { + afterEach(() => { + delete globalThis.fetch; }); -}); -describe('reading with the content source resolved', () => { describe('an html read on a source-bus site', () => { - it('reads from the base the config named', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + it('reads from the source bus', async () => { + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -256,7 +93,7 @@ describe('reading with the content source resolved', () => { }); it('composes the source-bus document, not da-admin\'s copy of it', async () => { - const { daSourceGet, env } = await build({ source: BUS_SOURCE }); + const { daSourceGet, env } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -265,7 +102,7 @@ describe('reading with the content source resolved', () => { }); it('forwards the author token to the source bus', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -276,7 +113,7 @@ describe('reading with the content source resolved', () => { describe('an html read on a legacy site', () => { it('reads from da-admin over the service binding', async () => { - const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE }); + const { daSourceGet, env, seen } = await build(); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -287,62 +124,68 @@ describe('reading with the content source resolved', () => { }); }); - describe('when the content source could not be resolved', () => { - // guessing reads past a migrated site's live page and hands the author its stale - // pre-migration copy, which the next save would then be based on - it('refuses an html read with 503 rather than guessing a store', async () => { - const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + describe('the store lookup on a read', () => { + it('happens once, and only the store it named is asked', async () => { + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual(res.status, 503); + assert.strictEqual(seen.lookups, 1); + assert.strictEqual(seen.legacy.length, 0); }); + }); - it('asks the caller to retry', async () => { - const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + describe('when the store cannot be reached at all', () => { + it('answers 503 rather than throwing on an html read', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.ok(Number(res.headers.get('Retry-After')) > 0); + assert.strictEqual(res.status, 503); }); - it('touches neither store', async () => { - const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + it('answers 503 rather than throwing on a non-html read', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); - assert.strictEqual(seen.bus.length, 0); - assert.strictEqual(seen.legacy.length, 0); + assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 503); }); - it('refuses a non-html read too', async () => { - const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + it('answers 503 rather than throwing on a HEAD', async () => { + const { daSourceHead, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - assert.strictEqual(res.status, 503); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + assert.strictEqual((await daSourceHead({ env, daCtx: getDaCtx(req) })).status, 503); }); - it('refuses a HEAD with 503 and no body', async () => { - const { daSourceHead, env } = await build({ source: UNKNOWN_SOURCE }); + it('answers 503 rather than throwing when da-admin is unreachable', async () => { + const { daSourceGet, env } = await build({ + legacy: () => { throw new TypeError('fetch failed'); }, + }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 503); - assert.strictEqual(await res.text(), ''); + assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 503); }); - it('asks the caller to retry on a HEAD too', async () => { - const { daSourceHead, env } = await build({ source: UNKNOWN_SOURCE }); + it('asks the caller to retry', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); assert.ok(Number(res.headers.get('Retry-After')) > 0); }); @@ -350,140 +193,126 @@ describe('reading with the content source resolved', () => { // the read refusals are rendered, by the preview iframe and by quick-edit, so they carry a // body that says what happened rather than an empty page it('says what happened, on both GET paths', async () => { - const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); const html = authedReq('https://main--site--org.ue.da.live/folder/content'); const asset = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); const htmlRes = await daSourceGet({ req: html, env, daCtx: getDaCtx(html) }); const assetRes = await daSourceGet({ req: asset, env, daCtx: getDaCtx(asset) }); - const htmlBody = await htmlRes.text(); - const assetBody = await assetRes.text(); - assert.match(htmlBody, /503/); - assert.match(assetBody, /503/); + assert.match(await htmlRes.text(), /503/); + assert.match(await assetRes.text(), /503/); }); - }); - describe('when the caller is not allowed to ask which store', () => { - it('answers 401 on a GET, so the client knows to re-authenticate', async () => { - const { daSourceGet, env, seen } = await build({ source: DENIED_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 401); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); - }); - - // the authorbus extension matches this sentinel exactly and recovers by refetching - // /gimme_cookie and refreshing the page - it('serves the da:401 shell the editor recovers from', async () => { - const { daSourceGet, env } = await build({ source: DENIED_SOURCE }); + it('answers 503 with no body on a HEAD', async () => { + const { daSourceHead, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); - assert.match(await res.text(), /content="da:401"/); + assert.strictEqual(await res.text(), ''); + assert.ok(Number(res.headers.get('Retry-After')) > 0); }); + }); - it('does not ask the caller to retry, since retrying cannot help', async () => { - const { daSourceGet, env } = await build({ source: DENIED_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + describe('what a store status means', () => { + // turning any of these into the blank starter template at HTTP 200 hands the author an + // empty document to save over a page that exists + [401, 403, 429, 500, 502].forEach((status) => { + it(`keeps the store's ${status} as the status`, async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => new Response('upstream said no', { status }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual(res.headers.get('Retry-After'), null); + assert.strictEqual(res.status, status); + }); }); - it('passes a 403 through as itself', async () => { - const { daSourceGet, env } = await build({ source: FORBIDDEN_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 403); - }); + // the body is the store's own except on a refusal, where it is replaced below + [429, 500, 502].forEach((status) => { + it(`passes the store's body through on a ${status}`, async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => new Response('upstream said no', { status }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - it('answers 401 on a non-html GET too', async () => { - const { daSourceGet, env } = await build({ source: DENIED_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 401); + assert.strictEqual(await res.text(), 'upstream said no'); + }); }); - it('answers 401 on a HEAD with no body', async () => { - const { daSourceHead, env } = await build({ source: DENIED_SOURCE }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + // /ping reads no token, so the store is the only thing to see one. the authorbus extension + // matches this sentinel exactly and recovers by refetching /gimme_cookie and refreshing + [401, 403].forEach((status) => { + it(`serves the da:401 shell when the store answers ${status} on an html read`, async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => new Response('', { status }), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual(res.status, 401); - assert.strictEqual(await res.text(), ''); + assert.strictEqual(res.status, status); + assert.match(await res.text(), /content="da:401"/); + }); }); - }); - describe('when the store cannot be reached at all', () => { - it('answers 503 rather than throwing on an html read', async () => { + it('does not ask the caller to retry a 401, since retrying cannot help', async () => { const { daSourceGet, env } = await build({ - source: BUS_SOURCE, - bus: () => { throw new TypeError('fetch failed'); }, + onSourceBus: true, + bus: () => new Response('', { status: 401 }), }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual(res.status, 503); + assert.strictEqual(res.headers.get('Retry-After'), null); }); - it('answers 503 rather than throwing on a non-html read', async () => { + it('passes a store 401 through bare on a non-html read, which renders nothing', async () => { const { daSourceGet, env } = await build({ - source: BUS_SOURCE, - bus: () => { throw new TypeError('fetch failed'); }, + onSourceBus: true, + bus: () => new Response('', { status: 401 }), }); const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); - assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 503); - }); - - it('answers 503 rather than throwing on a HEAD', async () => { - const { daSourceHead, env } = await build({ - source: BUS_SOURCE, - bus: () => { throw new TypeError('fetch failed'); }, - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - assert.strictEqual((await daSourceHead({ env, daCtx: getDaCtx(req) })).status, 503); + assert.strictEqual(res.status, 401); + assert.strictEqual(await res.text(), ''); }); - it('answers 503 rather than throwing when da-admin is unreachable', async () => { - const { daSourceGet, env } = await build({ - source: LEGACY_SOURCE, - legacy: () => { throw new TypeError('fetch failed'); }, + it('answers a store 401 on a HEAD with no body', async () => { + const { daSourceHead, env } = await build({ + onSourceBus: true, + bus: () => new Response('', { status: 401 }), }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - assert.strictEqual((await daSourceGet({ req, env, daCtx: getDaCtx(req) })).status, 503); - }); - }); - - describe('what a store status means', () => { - // turning any of these into the blank starter template at HTTP 200 hands the author an - // empty document to save over a page that exists - [401, 403, 429, 500, 502].forEach((status) => { - it(`passes a ${status} from the store through as itself`, async () => { - const { daSourceGet, env } = await build({ - source: BUS_SOURCE, - bus: () => new Response('upstream said no', { status }), - }); - const req = authedReq('https://main--site--org.ue.da.live/folder/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); - assert.strictEqual(res.status, status); - }); + assert.strictEqual(res.status, 401); + assert.strictEqual(await res.text(), ''); }); - it('composes the starter template on a 404, which is the one absent answer', async () => { - const { daSourceGet, env } = await build({ - source: BUS_SOURCE, + // a wrong yes from /ping produces this, and there is no second store to retry against: the + // author is handed the starter template over whatever da-admin still holds + it('composes the starter template on a 404 without asking the other store', async () => { + const { daSourceGet, env, seen } = await build({ + onSourceBus: true, bus: () => new Response('', { status: 404 }), }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); @@ -492,12 +321,13 @@ describe('reading with the content source resolved', () => { assert.strictEqual(res.status, 200); assert.ok(!(await res.text()).includes('from the source bus')); + assert.strictEqual(seen.legacy.length, 0); }); }); describe('UE instrumentation', () => { it('is applied on a UE host', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -506,7 +336,7 @@ describe('reading with the content source resolved', () => { }); it('is not applied on a preview host', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.preview.da.live/folder/content'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -533,13 +363,7 @@ describe('reading with the content source resolved', () => { daadmin: { fetch: async () => new Response('', { status: 200 }) }, }; const { daSourceGet } = await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { - default: async () => BUS_SOURCE, - fastSourceBus: async () => undefined, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', - }, + '../../src/storage/source-bus.js': { default: async () => true }, '../../src/utils/aemCtx.js': { getAemCtx: () => ({ ueHostname: 'ue.da.live', previewUrl: 'https://p.example' }), getAEMHtml: async () => '', @@ -557,7 +381,7 @@ describe('reading with the content source resolved', () => { describe('a non-html read', () => { it('goes to the source bus fully normalized', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -566,7 +390,7 @@ describe('reading with the content source resolved', () => { }); it('goes to da-admin fully lowercased on a legacy site', async () => { - const { daSourceGet, env, seen } = await build({ source: LEGACY_SOURCE }); + const { daSourceGet, env, seen } = await build(); const req = authedReq('https://main--site--org.ue.da.live/Media/Holiday.PNG'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -577,7 +401,7 @@ describe('reading with the content source resolved', () => { describe('a HEAD', () => { it('goes to the source bus on a source-bus site', async () => { - const { daSourceHead, env, seen } = await build({ source: BUS_SOURCE }); + const { daSourceHead, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceHead({ env, daCtx: getDaCtx(req) }); @@ -588,7 +412,7 @@ describe('reading with the content source resolved', () => { }); it('goes to da-admin on a legacy site', async () => { - const { daSourceHead, env, seen } = await build({ source: LEGACY_SOURCE }); + const { daSourceHead, env, seen } = await build(); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); await daSourceHead({ env, daCtx: getDaCtx(req) }); @@ -599,7 +423,7 @@ describe('reading with the content source resolved', () => { it('passes a 404 from the store through, since HEAD composes nothing', async () => { const { daSourceHead, env } = await build({ - source: BUS_SOURCE, + onSourceBus: true, bus: () => new Response('', { status: 404 }), }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); @@ -610,21 +434,9 @@ describe('reading with the content source resolved', () => { }); }); - describe('a read that cannot be placed at all', () => { - it('never asks a store when the hostname named no site', async () => { - const { daSourceGet, env, seen } = await build({ source: UNKNOWN_SOURCE }); - const req = authedReq('https://xyz.ue.da.live/content'); - - const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); - - assert.strictEqual(res.status, 503); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); - }); - }); - describe('a media read, which the handlers race against the AEM proxy', () => { // the store answer is preferred at 200 and the published copy otherwise, so an image that is - // published still resolves during a lookup outage. But when neither answers, a 404 says "this + // published still resolves during a store outage. But when neither answers, a 404 says "this // image does not exist" where the truth is "we could not find out", so the store's own 503 // wins over a non-200 from the proxy. const raced = async (handler, storeStatus, aemStatus) => { @@ -642,7 +454,7 @@ describe('reading with the content source resolved', () => { }; ['get', 'head'].forEach((handler) => { - it(`prefers the published copy over an unresolved store on a ${handler.toUpperCase()}`, async () => { + it(`prefers the published copy over an unreachable store on a ${handler.toUpperCase()}`, async () => { assert.strictEqual(await raced(handler, 503, 200), 200); }); @@ -660,10 +472,10 @@ describe('reading with the content source resolved', () => { }); // getHandler races an image read against *.aem.page and takes the proxy answer whenever the - // store read is not a 200. So an unresolved source degrades an image to the published copy + // store read is not a 200. So an unreachable store degrades an image to the published copy // rather than breaking the page, and an image cannot be laundered into a write: a POST to a // non-html path is refused with 415 before anything is resolved. - it('falls through to the AEM proxy for an image when the source is unresolved', async () => { + it('falls through to the AEM proxy for an image when the store did not answer', async () => { const seen = []; globalThis.fetch = async (input) => { seen.push(input.toString()); @@ -687,8 +499,11 @@ describe('reading with the content source resolved', () => { // mp4 is not in the raced extensions, so it has no published copy to fall back to and the // refusal reaches the caller as itself - it('refuses a video read when the source is unresolved', async () => { - const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE }); + it('refuses a video read when the store could not be reached', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); const req = authedReq('https://main--site--org.ue.da.live/folder/clip.mp4'); const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -696,8 +511,8 @@ describe('reading with the content source resolved', () => { assert.strictEqual(res.status, 503); }); - it('reads a video from the source bus when the source is known', async () => { - const { daSourceGet, env, seen } = await build({ source: BUS_SOURCE }); + it('reads a video from the source bus on a source-bus site', async () => { + const { daSourceGet, env, seen } = await build({ onSourceBus: true }); const req = authedReq('https://main--site--org.ue.da.live/folder/clip.mp4'); await daSourceGet({ req, env, daCtx: getDaCtx(req) }); @@ -708,8 +523,12 @@ describe('reading with the content source resolved', () => { describe('the order of the two things that can fail', () => { // a missing AEM branch is answered as it was before, so quick-edit still gets its shell - it('reports a missing AEM branch even when the source is unresolved', async () => { - const { daSourceGet, env } = await build({ source: UNKNOWN_SOURCE, headHtml: undefined }); + it('reports a missing AEM branch even when the store did not answer', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + headHtml: undefined, + bus: () => { throw new TypeError('fetch failed'); }, + }); const req = authedReq('https://main--site--org.ue.da.live/folder/content'); const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 59b3dbb3..90af547d 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -14,11 +14,7 @@ import assert from 'assert'; import esmock from 'esmock'; import { getDaCtx } from '../../src/utils/daCtx.js'; - -const LEGACY_SOURCE = { kind: 'legacy' }; -const BUS_SOURCE = { kind: 'sourcebus', base: 'https://api.aem.live/org/sites/site/source' }; -const UNKNOWN_SOURCE = { kind: 'unknown', reason: 'the API answered 503' }; -const DENIED_SOURCE = { kind: 'unauthorized', status: 401 }; +import { SOURCE_BUS_READ_ONLY_MESSAGE } from '../../src/utils/constants.js'; const AT = 'https://main--site--org.ue.da.live/folder/content'; const DOC = '

the author typed this

'; @@ -30,20 +26,28 @@ const uePost = (url, html = DOC) => { return new Request(url, { method: 'POST', body, headers: { Authorization: 'Bearer t' } }); }; -const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { - const seen = { bus: [], legacy: [], lookups: 0 }; +const build = async ({ onSourceBus = false, status = 201 } = {}) => { + const seen = { + bus: [], legacy: [], lookups: 0, order: [], + }; const capture = async (request) => { - const clone = request.clone(); + const contentType = request.headers.get('Content-Type'); + const [body, form] = await Promise.all([ + request.clone().text(), + contentType?.startsWith('multipart/') ? request.clone().formData() : undefined, + ]); return { url: request.url, method: request.method, headers: request.headers, - body: await clone.text(), - contentType: request.headers.get('Content-Type'), + body, + form, + contentType, }; }; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); + seen.order.push('store'); seen.bus.push(await capture(request)); return new Response('', { status }); }; @@ -53,20 +57,19 @@ const build = async ({ source = LEGACY_SOURCE, status = 201 } = {}) => { daadmin: { fetch: async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); + seen.order.push('store'); seen.legacy.push(await capture(request)); return new Response('', { status }); }, }, }; const mod = await esmock('../../src/routes/da-admin.js', { - '../../src/storage/content-source.js': { + '../../src/storage/source-bus.js': { default: async () => { seen.lookups += 1; - return source; + seen.order.push('lookup'); + return onSourceBus; }, - SOURCE_BUS: 'sourcebus', - LEGACY: 'legacy', - UNKNOWN: 'unknown', }, }); return { daSourcePost: mod.daSourcePost, env, seen }; @@ -79,77 +82,111 @@ const post = async (opts, url = AT) => { return { res, seen }; }; -afterEach(() => { - delete globalThis.fetch; -}); - describe('writing to the store that holds the site', () => { + afterEach(() => { + delete globalThis.fetch; + }); + + // the Universal Editor on da.live has never supported source-bus sites, and da-admin would take + // the document at 201 for a key nothing serves, so the write is refused rather than misplaced describe('a source-bus site', () => { - it('writes to the base the config named', async () => { - const { seen } = await post({ source: BUS_SOURCE }); + it('is refused with 405 and touches neither store', async () => { + const { res, seen } = await post({ onSourceBus: true }); + assert.strictEqual(res.status, 405); + assert.strictEqual(seen.bus.length, 0); assert.strictEqual(seen.legacy.length, 0); - assert.strictEqual(seen.bus.length, 1); - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); }); - it('sends the document as the raw body the source bus parses', async () => { - const { seen } = await post({ source: BUS_SOURCE }); + it('names the methods that are left', async () => { + const { res } = await post({ onSourceBus: true }); + + assert.strictEqual(res.headers.get('Allow'), 'GET, HEAD, OPTIONS'); + }); + + it('does not ask the caller to retry, since retrying cannot help', async () => { + const { res } = await post({ onSourceBus: true }); - assert.strictEqual(seen.bus[0].body, DOC); - assert.strictEqual(seen.bus[0].contentType, 'text/html'); + assert.strictEqual(res.headers.get('Retry-After'), null); }); - // helix-api-service parses no form data; the envelope would be stored as the document text - // and answered 201 - it('never wraps the body in a multipart envelope', async () => { - const { seen } = await post({ source: BUS_SOURCE }); + // nothing renders a POST body, and UES embeds it verbatim in its problem+json error string, + // so the exact text is what the author is shown + it('says what happened in plain text', async () => { + const { res } = await post({ onSourceBus: true }); + + assert.match(res.headers.get('Content-Type'), /^text\/plain/); + assert.strictEqual(await res.text(), SOURCE_BUS_READ_ONLY_MESSAGE); + }); + }); + + describe('a legacy site', () => { + it('writes to da-admin over the service binding', async () => { + const { seen } = await post({}); + + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy.length, 1); + assert.strictEqual(seen.legacy[0].method, 'POST'); + assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); + }); + + // da-admin parses the document out of a `data` form part and ignores a raw body + it('sends the document as a text/html data part', async () => { + const { seen } = await post({}); + + const data = seen.legacy[0].form.get('data'); - assert.ok(!seen.bus[0].body.includes('Content-Disposition'), seen.bus[0].body); - assert.ok(!/boundary/i.test(seen.bus[0].contentType ?? ''), seen.bus[0].contentType); + assert.strictEqual(await data.text(), DOC); + assert.strictEqual(data.type, 'text/html'); }); it('authorizes with the caller token', async () => { - const { seen } = await post({ source: BUS_SOURCE }); + const { seen } = await post({}); - assert.strictEqual(seen.bus[0].headers.get('Authorization'), 'Bearer t'); + assert.strictEqual(seen.legacy[0].headers.get('Authorization'), 'Bearer t'); }); it('passes the store answer back', async () => { - const { res } = await post({ source: BUS_SOURCE, status: 412 }); + const { res } = await post({ status: 412 }); assert.strictEqual(res.status, 412); }); - }); - describe('a legacy site', () => { - it('writes to da-admin as a data form part', async () => { - const { seen } = await post({ source: LEGACY_SOURCE }); + it('normalizes the whole path', async () => { + const { seen } = await post( + {}, + 'https://main--site--org.ue.da.live/Folder/Content', + ); - assert.strictEqual(seen.bus.length, 0); assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); - assert.ok(seen.legacy[0].body.includes('name="data"'), seen.legacy[0].body); - assert.ok(seen.legacy[0].body.includes('the author typed this'), seen.legacy[0].body); + }); + + it('strips the UE data attributes before the store sees them', async () => { + const { daSourcePost, env, seen } = await build({}); + const req = uePost( + AT, + '

text

', + ); + + await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + + assert.ok(!seen.legacy[0].body.includes('data-aue-resource'), seen.legacy[0].body); }); }); - describe('no write carries a precondition', () => { - // only the source bus sets an etag on a read, and nothing round-trips it into the save: the - // editor keeps the connection uri it was served at page load and posts back to it for every - // edit, so a version pin would land the first save and refuse the rest with 412 - [['a source-bus', BUS_SOURCE, 'bus'], ['a legacy', LEGACY_SOURCE, 'legacy']].forEach( - ([what, source, where]) => { - it(`${what} write sends no If-Match or If-None-Match`, async () => { - const { seen } = await post({ source }); - - assert.strictEqual(seen[where][0].headers.get('If-Match'), null); - assert.strictEqual(seen[where][0].headers.get('If-None-Match'), null); - }); - }, - ); + describe('a legacy write carries no precondition', () => { + // nothing round-trips an etag into the save: the editor keeps the connection uri it was + // served at page load and posts back to it for every edit, so a version pin would land the + // first save and refuse the rest with 412 + it('sends no If-Match or If-None-Match', async () => { + const { seen } = await post({}); + + assert.strictEqual(seen.legacy[0].headers.get('If-Match'), null); + assert.strictEqual(seen.legacy[0].headers.get('If-None-Match'), null); + }); it('so a UE session can save the same page many times', async () => { - const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); + const { daSourcePost, env, seen } = await build({}); const statuses = []; for (let i = 0; i < 4; i += 1) { @@ -160,63 +197,30 @@ describe('writing to the store that holds the site', () => { } assert.deepStrictEqual(statuses, [201, 201, 201, 201]); - assert.strictEqual(seen.bus.length, 4); + assert.strictEqual(seen.legacy.length, 4); }); }); describe('the store lookup on a write', () => { - // nothing is held between requests, so a write and the read before it resolve independently - it('happens once per write', async () => { - const { seen } = await post({ source: BUS_SOURCE }); - - assert.strictEqual(seen.lookups, 1); - }); - + // a legacy write is the case that can tell the two orderings apart: the store is reached + // either way, so only the sequence says whether the write went out before it was placed it('happens before anything is sent to a store', async () => { - const { seen } = await post({ source: UNKNOWN_SOURCE }); - - assert.strictEqual(seen.lookups, 1); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); - }); - }); - - describe('when the content source could not be resolved', () => { - it('refuses with 503 and touches neither store', async () => { - const { res, seen } = await post({ source: UNKNOWN_SOURCE }); - - assert.strictEqual(res.status, 503); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); - }); + const { seen } = await post({}); - it('asks the caller to retry', async () => { - const { res } = await post({ source: UNKNOWN_SOURCE }); - - assert.ok(Number(res.headers.get('Retry-After')) > 0); - }); - - // nothing renders a POST body, and UES embeds it verbatim in its problem+json error string - it('says so in plain text', async () => { - const { res } = await post({ source: UNKNOWN_SOURCE }); - - assert.match(res.headers.get('Content-Type'), /^text\/plain/); - assert.ok((await res.text()).length > 0); + assert.deepStrictEqual(seen.order, ['lookup', 'store']); }); - }); - describe('when the caller is not allowed to ask which store', () => { - it('answers 401, not a retryable 503', async () => { - const { res, seen } = await post({ source: DENIED_SOURCE }); + it('happens on a source-bus site too, which is what the refusal rests on', async () => { + const { seen } = await post({ onSourceBus: true }); - assert.strictEqual(res.status, 401); - assert.strictEqual(res.headers.get('Retry-After'), null); - assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + assert.strictEqual(seen.lookups, 1); }); }); - describe('when the store cannot be reached at all', () => { + describe('when da-admin cannot be reached at all', () => { it('answers 503 rather than throwing', async () => { - const { daSourcePost, env } = await build({ source: BUS_SOURCE }); - globalThis.fetch = async () => { + const { daSourcePost, env } = await build({}); + env.daadmin.fetch = async () => { throw new TypeError('fetch failed'); }; const req = uePost(AT); @@ -227,21 +231,11 @@ describe('writing to the store that holds the site', () => { }); }); - describe('what is written', () => { - it('strips the UE data attributes before the store sees them', async () => { - const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); - const req = uePost( - AT, - '

text

', - ); - - await daSourcePost({ req, env, daCtx: getDaCtx(req) }); - - assert.ok(!seen.bus[0].body.includes('data-aue-resource'), seen.bus[0].body); - }); - - it('refuses a non-html path before resolving anything', async () => { - const { daSourcePost, env, seen } = await build({ source: BUS_SOURCE }); + describe('a non-html path', () => { + // driven on a source-bus site, so the 415 has to come from the extension check rather than + // from the refusal below it. on a legacy site either ordering would pass. + it('is refused before anything is resolved', async () => { + const { daSourcePost, env, seen } = await build({ onSourceBus: true }); const req = uePost('https://main--site--org.ue.da.live/folder/data.json'); const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); @@ -251,24 +245,4 @@ describe('writing to the store that holds the site', () => { assert.strictEqual(seen.bus.length + seen.legacy.length, 0); }); }); - - describe('the path a save is written to', () => { - it('is stored under the normalized path on the source bus', async () => { - const { seen } = await post( - { source: BUS_SOURCE }, - 'https://main--site--org.ue.da.live/Folder/Content', - ); - - assert.strictEqual(seen.bus[0].url, 'https://api.aem.live/org/sites/site/source/folder/content.html'); - }); - - it('normalizes the whole path for da-admin', async () => { - const { seen } = await post( - { source: LEGACY_SOURCE }, - 'https://main--site--org.ue.da.live/Folder/Content', - ); - - assert.strictEqual(seen.legacy[0].url, 'https://admin.da.live/source/org/site/folder/content.html'); - }); - }); }); diff --git a/test/storage/content-source.test.js b/test/storage/content-source.test.js deleted file mode 100644 index be02e724..00000000 --- a/test/storage/content-source.test.js +++ /dev/null @@ -1,404 +0,0 @@ -/* - * Copyright 2026 Adobe. All rights reserved. - * This file is licensed to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy - * of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS - * OF ANY KIND, either express or implied. See the License for the specific language - * governing permissions and limitations under the License. - */ - -/* eslint-env mocha */ -import assert from 'assert'; - -const { - default: resolveContentSource, fastSourceBus, -} = await import('../../src/storage/content-source.js'); - -const env = { AEM_API: 'https://api.aem.live', HLX_ADMIN: 'https://admin.hlx.page' }; - -const daCtx = (over = {}) => ({ - org: 'org', site: 'site', ref: 'main', authToken: 'Bearer t', ...over, -}); - -let calls; - -const stubFetch = (respond) => { - calls = []; - globalThis.fetch = async (input, init) => { - calls.push({ url: input.toString(), init }); - return respond(input.toString(), init); - }; -}; - -const sidekick = (contentSourceUrl) => new Response( - JSON.stringify({ contentSourceUrl, contentSourceType: 'markup' }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, -); - -const legacyBody = () => sidekick('https://content.da.live/org/site/'); - -describe('resolveContentSource', () => { - afterEach(() => { - delete globalThis.fetch; - }); - - describe('the request it makes', () => { - it('asks the AEM API for the site', async () => { - stubFetch(legacyBody); - - await resolveContentSource(env, daCtx()); - - assert.strictEqual(calls.length, 1); - assert.strictEqual(calls[0].url, 'https://api.aem.live/org/sites/site/sidekick'); - }); - - // both stores read the same config service and the source is per site, so the branch does - // not change the answer - it('does not vary by ref', async () => { - stubFetch(legacyBody); - - await resolveContentSource(env, daCtx({ ref: 'branch' })); - - assert.strictEqual(calls[0].url, 'https://api.aem.live/org/sites/site/sidekick'); - }); - - it('passes the author token on, so a private site resolves', async () => { - stubFetch(legacyBody); - - await resolveContentSource(env, daCtx()); - - assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), 'Bearer t'); - }); - - // a config service that accepts the connection and never answers would otherwise hold the - // request open for as long as the platform allows - it('gives up on the lookup rather than hanging', async () => { - stubFetch(legacyBody); - - await resolveContentSource(env, daCtx()); - - const { signal } = calls[0].init; - assert.ok(signal, 'the lookup carries an abort signal'); - assert.strictEqual(typeof signal.aborted, 'boolean'); - }); - - it('asks anyway when there is no author token', async () => { - stubFetch(legacyBody); - - await resolveContentSource(env, daCtx({ authToken: undefined })); - - assert.strictEqual(calls.length, 1); - assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), null); - }); - }); - - describe('when the content source is on api.aem.live', () => { - it('answers sourcebus', async () => { - stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'sourcebus'); - }); - - it('carries the base url from the config, rather than rebuilding it', async () => { - stubFetch(() => sidekick('https://api.aem.live/other/sites/elsewhere/source')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.base, 'https://api.aem.live/other/sites/elsewhere/source'); - }); - - it('drops a trailing slash on the base, so paths do not double up', async () => { - stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source/')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.base, 'https://api.aem.live/org/sites/site/source'); - }); - }); - - describe('when the content source is on content.da.live', () => { - it('answers legacy', async () => { - stubFetch(legacyBody); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'legacy'); - }); - }); - - describe('when the answer is not one of the two stores', () => { - // this worker only serves DA-backed sites, so a google or onedrive mount is not - // something either store can answer for and must not be guessed at - it('answers unknown for a source url it does not recognise', async () => { - stubFetch(() => sidekick('https://drive.google.com/drive/folders/abc')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown when contentSourceUrl is missing from the body', async () => { - stubFetch(() => new Response(JSON.stringify({ project: 'site' }), { status: 200 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown for a host that only starts like the API', async () => { - stubFetch(() => sidekick('https://api.aem.live.evil.example/org/sites/site/source')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - // the base is used verbatim as a store url and the author token goes with it, so the store - // has to be named at the front of the url and not merely somewhere inside it - it('answers unknown when a store url appears anywhere but the start', async () => { - stubFetch(() => sidekick('https://elsewhere.example/?to=https://api.aem.live/org/sites/site/source')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown for a legacy url buried mid-string too', async () => { - stubFetch(() => sidekick('https://elsewhere.example/#https://content.da.live/org/site/')); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - }); - - describe('when the question could not be answered', () => { - // a 404 is what the sidekick route returns when config resolution produced nothing - // (helix-api-service and helix-admin both: `if (config) { ... } return { status: 404 }`), so - // it means "we do not know", not "legacy" - it('answers unknown on a 404', async () => { - stubFetch(() => new Response('', { status: 404 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown on a 5xx', async () => { - stubFetch(() => new Response('', { status: 503 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown when the body is not json', async () => { - stubFetch(() => new Response('gateway', { status: 200 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('answers unknown when the fetch throws', async () => { - stubFetch(() => { - throw new Error('boom'); - }); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - - it('says why it could not answer', async () => { - stubFetch(() => new Response('', { status: 404 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.match(source.reason, /404/); - }); - }); - - describe('when the caller is not allowed to ask', () => { - // "we do not know" is retryable and "you are not authenticated" is not. Reporting an expired - // session as unknown turns into a 503 that says retry, so the client never re-authenticates - // and the da:401 recovery the authorbus extension has never fires. - [401, 403].forEach((status) => { - it(`answers unauthorized on a ${status}, not unknown`, async () => { - stubFetch(() => new Response('', { status })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unauthorized'); - }); - - it(`carries the ${status} through, so the caller answers the same`, async () => { - stubFetch(() => new Response('', { status })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.status, status); - }); - }); - - it('still answers unknown for a 5xx, which is retryable', async () => { - stubFetch(() => new Response('', { status: 502 })); - - const source = await resolveContentSource(env, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - }); - - describe('when there is no site to ask about', () => { - // either one missing is enough: a half-parsed request would otherwise build a url with - // "undefined" in it and send the author's token to it - [ - ['neither', { org: undefined, site: undefined }], - ['no org', { org: undefined }], - ['no site', { site: undefined }], - ['an empty org', { org: '' }], - ['an empty site', { site: '' }], - ].forEach(([what, over]) => { - it(`answers unknown without making a request: ${what}`, async () => { - stubFetch(legacyBody); - - const source = await resolveContentSource(env, daCtx(over)); - - assert.strictEqual(source.kind, 'unknown'); - assert.strictEqual(calls.length, 0); - }); - }); - }); - - describe('the API host', () => { - it('comes from env, so stage can point elsewhere', async () => { - stubFetch(legacyBody); - - await resolveContentSource({ AEM_API: 'https://api.stage.example' }, daCtx()); - - assert.strictEqual(calls[0].url, 'https://api.stage.example/org/sites/site/sidekick'); - }); - - it('tolerates a trailing slash on it', async () => { - stubFetch(legacyBody); - - await resolveContentSource({ AEM_API: 'https://api.aem.live/' }, daCtx()); - - assert.strictEqual(calls[0].url, 'https://api.aem.live/org/sites/site/sidekick'); - }); - - // the same env value decides where we ask and what counts as the source bus, so pointing at - // stage must not leave the prefix test matching production - it('is also what makes a source url the source bus', async () => { - stubFetch(() => sidekick('https://api.aem.live/org/sites/site/source')); - - const source = await resolveContentSource({ AEM_API: 'https://api.stage.example' }, daCtx()); - - assert.strictEqual(source.kind, 'unknown'); - }); - }); -}); - -describe('fastSourceBus', () => { - afterEach(() => { - delete globalThis.fetch; - }); - - // /ping answers an enrolled site from the Fastly edge dictionary in ~37ms without reaching an - // origin, where the config read is ~529ms and always reaches one. Its `true` is positive - // evidence; its absence conflates legacy, a config that would not resolve, and a site that does - // not exist, so only the yes is usable. - const ping = (headers = {}, status = 200) => new Response('', { status, headers }); - - it('asks /ping on the admin host', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - await fastSourceBus(env, daCtx()); - - assert.strictEqual(calls.length, 1); - assert.strictEqual(calls[0].url, 'https://admin.hlx.page/ping/org/site'); - }); - - // /ping is exempt from authorize() in helix-admin and answers the same with or without a token - it('sends no token, since /ping does not read one', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - await fastSourceBus(env, daCtx()); - - assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), null); - }); - - it('gives up rather than hanging', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - await fastSourceBus(env, daCtx()); - - assert.ok(calls[0].init.signal, 'the probe carries an abort signal'); - }); - - describe('when /ping says the site is upgraded', () => { - it('answers sourcebus', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - assert.strictEqual((await fastSourceBus(env, daCtx())).kind, 'sourcebus'); - }); - - // helix-api-service parses org and site out of the source url and 400s unless both match the - // request's own (src/contentproxy/source/utils.js, "only allow source bus from the same org - // and site"), so this is the only base a source-bus site can legally have - it('builds the only base that org and site can legally have', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - const source = await fastSourceBus(env, daCtx()); - - assert.strictEqual(source.base, 'https://api.aem.live/org/sites/site/source'); - }); - - it('builds it on AEM_API, so stage moves it', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - const source = await fastSourceBus({ ...env, AEM_API: 'https://api.stage.example' }, daCtx()); - - assert.strictEqual(source.base, 'https://api.stage.example/org/sites/site/source'); - }); - }); - - describe('when /ping does not say so', () => { - [ - ['the header is absent', {}, 200], - ['the header is false', { 'x-api-upgrade-available': 'false' }, 200], - ['the header is empty', { 'x-api-upgrade-available': '' }, 200], - ['the header is TRUE in capitals', { 'x-api-upgrade-available': 'TRUE' }, 200], - ['the status is 404', { 'x-api-upgrade-available': 'true' }, 404], - ['the status is 405', {}, 405], - ['the status is 500', {}, 500], - ].forEach(([what, headers, status]) => { - it(`answers undefined: ${what}`, async () => { - stubFetch(() => ping(headers, status)); - - assert.strictEqual(await fastSourceBus(env, daCtx()), undefined); - }); - }); - - it('answers undefined when the probe throws', async () => { - stubFetch(() => { - throw new TypeError('fetch failed'); - }); - - assert.strictEqual(await fastSourceBus(env, daCtx()), undefined); - }); - - it('answers undefined without asking when there is no site', async () => { - stubFetch(() => ping({ 'x-api-upgrade-available': 'true' })); - - assert.strictEqual(await fastSourceBus(env, daCtx({ site: undefined })), undefined); - assert.strictEqual(calls.length, 0); - }); - }); -}); diff --git a/test/storage/source-bus.test.js b/test/storage/source-bus.test.js new file mode 100644 index 00000000..17c3691f --- /dev/null +++ b/test/storage/source-bus.test.js @@ -0,0 +1,179 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; + +const { default: isSourceBus } = await import('../../src/storage/source-bus.js'); + +const env = { AEM_API: 'https://api.aem.live', HLX_ADMIN: 'https://admin.hlx.page' }; + +const daCtx = (over = {}) => ({ + org: 'org', site: 'site', ref: 'main', authToken: 'Bearer t', ...over, +}); + +let calls; + +const stubFetch = (respond) => { + calls = []; + globalThis.fetch = async (input, init) => { + calls.push({ url: input.toString(), init }); + return respond(input.toString(), init); + }; +}; + +const ping = (headers = {}, status = 200) => new Response('', { status, headers }); +const upgraded = () => ping({ 'x-api-upgrade-available': 'true' }); + +describe('isSourceBus', () => { + afterEach(() => { + delete globalThis.fetch; + }); + + describe('the request it makes', () => { + it('asks /ping on the admin host', async () => { + stubFetch(upgraded); + + await isSourceBus(env, daCtx()); + + assert.strictEqual(calls.length, 1); + assert.strictEqual(calls[0].url, 'https://admin.hlx.page/ping/org/site'); + }); + + it('takes the admin host from env, so stage can point elsewhere', async () => { + stubFetch(upgraded); + + await isSourceBus({ ...env, HLX_ADMIN: 'https://admin.stage.example' }, daCtx()); + + assert.strictEqual(calls[0].url, 'https://admin.stage.example/ping/org/site'); + }); + + // both stores read one config service and the source is per site, so the branch cannot change + // the answer + it('does not vary by ref', async () => { + stubFetch(upgraded); + + await isSourceBus(env, daCtx({ ref: 'branch' })); + + assert.strictEqual(calls[0].url, 'https://admin.hlx.page/ping/org/site'); + }); + + // /ping is exempt from authorize() in helix-admin and answers the same with or without a token + it('sends no token, since /ping does not read one', async () => { + stubFetch(upgraded); + + await isSourceBus(env, daCtx()); + + assert.strictEqual(new Headers(calls[0].init.headers).get('Authorization'), null); + }); + + it('gives up rather than hanging', async () => { + stubFetch(upgraded); + + await isSourceBus(env, daCtx()); + + assert.ok(calls[0].init.signal, 'the probe carries an abort signal'); + }); + }); + + describe('when /ping says the site is upgraded', () => { + it('answers true', async () => { + stubFetch(upgraded); + + assert.strictEqual(await isSourceBus(env, daCtx()), true); + }); + + // presence, not value: da-nx tests the same header with `!== null` (nx2/utils/api.js, + // isHlx6), and two clients reading it differently would split one site across two stores + ['false', '', 'TRUE'].forEach((value) => { + it(`counts any value, including ${JSON.stringify(value)}`, async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': value })); + + assert.strictEqual(await isSourceBus(env, daCtx()), true); + }); + }); + + // no status test, for the same reason. the edge sets the header from its dictionary, so a + // rate-limited or erroring origin behind it does not make an enrolled site legacy + [429, 500, 503].forEach((status) => { + it(`counts it on a ${status}, since the header is what carries the answer`, async () => { + stubFetch(() => ping({ 'x-api-upgrade-available': 'true' }, status)); + + assert.strictEqual(await isSourceBus(env, daCtx()), true); + }); + }); + }); + + describe('when /ping does not say so', () => { + [ + ['the header is absent', {}, 200], + ['the header is absent on a 404', {}, 404], + ['the header is absent on a 405', {}, 405], + ['the header is absent on a 500', {}, 500], + ].forEach(([what, headers, status]) => { + it(`answers false: ${what}`, async () => { + stubFetch(() => ping(headers, status)); + + assert.strictEqual(await isSourceBus(env, daCtx()), false); + }); + }); + + it('answers false when the probe throws', async () => { + stubFetch(() => { + throw new TypeError('fetch failed'); + }); + + assert.strictEqual(await isSourceBus(env, daCtx()), false); + }); + + // HLX_ADMIN is a wrangler.toml constant, so this is a broken deploy rather than a runtime + // condition. It answers legacy instead of throwing out of every read. + it('answers false without asking when HLX_ADMIN is unusable', async () => { + stubFetch(upgraded); + + assert.strictEqual(await isSourceBus({ AEM_API: 'https://api.aem.live' }, daCtx()), false); + assert.strictEqual(calls.length, 0); + }); + }); + + describe('when there is no site to ask about', () => { + // either one missing is enough: a half-parsed request would otherwise build a ping url with + // "undefined" in it + [ + ['neither', { org: undefined, site: undefined }], + ['no org', { org: undefined }], + ['no site', { site: undefined }], + ['an empty org', { org: '' }], + ['an empty site', { site: '' }], + ].forEach(([what, over]) => { + it(`answers false without making a request: ${what}`, async () => { + stubFetch(upgraded); + + assert.strictEqual(await isSourceBus(env, daCtx(over)), false); + assert.strictEqual(calls.length, 0); + }); + }); + }); + + // nothing is remembered between calls, so an enrolment takes effect on the next read and a + // config blip cannot pin a stale answer + it('probes every time it is asked', async () => { + let enrolled = false; + stubFetch(() => (enrolled ? upgraded() : ping())); + + assert.strictEqual(await isSourceBus(env, daCtx()), false); + enrolled = true; + + assert.strictEqual(await isSourceBus(env, daCtx()), true); + assert.strictEqual(calls.length, 2); + }); +}); diff --git a/test/storage/store.test.js b/test/storage/store.test.js index b1c2835b..62a5025b 100644 --- a/test/storage/store.test.js +++ b/test/storage/store.test.js @@ -13,42 +13,46 @@ /* eslint-env mocha */ import assert from 'assert'; import { getDaCtx } from '../../src/utils/daCtx.js'; -import { LEGACY, SOURCE_BUS } from '../../src/storage/content-source.js'; const { default: getStore } = await import('../../src/storage/store.js'); -const env = { DA_ADMIN: 'https://admin.da.live' }; +const env = { DA_ADMIN: 'https://admin.da.live', AEM_API: 'https://api.aem.live' }; const ctxFor = (url) => getDaCtx(new Request(url, { headers: { Authorization: 'Bearer t' } })); -const legacy = { kind: LEGACY }; -const bus = { kind: SOURCE_BUS, base: 'https://api.aem.live/org/sites/site/source' }; describe('getStore', () => { - describe('the url it reads and writes', () => { + describe('the url a document has in the store', () => { it('builds the source-bus url from the normalized path, not the requested one', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'), bus); + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Media/Holiday.PNG'), true); assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/media/holiday.png'); }); it('builds a legacy url under DA_ADMIN from the lowercased source path', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Folder/Doc'), legacy); + const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/Folder/Doc'), false); assert.strictEqual(store.url.toString(), 'https://admin.da.live/source/org/site/folder/doc.html'); }); - it('builds a source-bus url on the base the config named', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/folder/doc'), bus); + // helix-api-service refuses a source url naming another org or site, so the request's own org + // and site are the only base it can have + it('builds the source-bus url from the request\'s own org and site', () => { + const store = getStore(env, ctxFor('https://main--other--shared.ue.da.live/folder/doc'), true); - assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/folder/doc.html'); + assert.strictEqual(store.url.toString(), 'https://api.aem.live/shared/sites/other/source/folder/doc.html'); + }); + + it('builds it on AEM_API, so stage moves it', () => { + const bound = { ...env, AEM_API: 'https://api.stage.example' }; + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/folder/doc'), true); + + assert.strictEqual(store.url.toString(), 'https://api.stage.example/org/sites/site/source/folder/doc.html'); }); - it('takes the base verbatim, so a config naming another org is followed', () => { - const store = getStore(env, ctxFor('https://main--site--org.ue.da.live/doc'), { - kind: SOURCE_BUS, - base: 'https://api.aem.live/shared/sites/library/source', - }); + it('tolerates a trailing slash on AEM_API', () => { + const bound = { ...env, AEM_API: 'https://api.aem.live/' }; + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/folder/doc'), true); - assert.strictEqual(store.url.toString(), 'https://api.aem.live/shared/sites/library/source/doc.html'); + assert.strictEqual(store.url.toString(), 'https://api.aem.live/org/sites/site/source/folder/doc.html'); }); }); @@ -60,7 +64,7 @@ describe('getStore', () => { return new Response(''); }; const bound = { ...env, daadmin: { fetch: record } }; - const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), legacy); + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), false); await store.fetch(store.url, { method: 'GET' }); @@ -74,7 +78,7 @@ describe('getStore', () => { return new Response(''); }; const bound = { ...env, daadmin: { fetch: async () => assert.fail('used the binding') } }; - const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), bus); + const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), true); await store.fetch(store.url, { method: 'GET' }); @@ -82,83 +86,4 @@ describe('getStore', () => { assert.strictEqual(seen.length, 1); }); }); - - describe('the write body each store parses', () => { - // both stores are captured with the same recorder, so a test names the store it expects by - // passing its kind rather than by picking a transport - const written = async (kind, html = '') => { - let sent; - const capture = async (input, init) => { - sent = input instanceof Request ? input : new Request(input, init); - return new Response(''); - }; - globalThis.fetch = capture; - const bound = { ...env, daadmin: { fetch: capture } }; - const store = getStore(bound, ctxFor('https://main--site--org.ue.da.live/doc'), kind); - - await store.write(html, 'Bearer t'); - - delete globalThis.fetch; - return sent; - }; - - // helix-api-service parses no form data anywhere: getValidPayload reads the raw buffer and - // types it from the path extension. Sending da-admin's multipart envelope stores the - // boundary lines as the document text and answers 201, so this is not a cosmetic difference. - it('sends the source bus the document as the raw body', async () => { - const sent = await written(bus, '

hi

'); - - assert.strictEqual(await sent.text(), '

hi

'); - }); - - it('types the source-bus write as text/html', async () => { - const sent = await written(bus); - - assert.strictEqual(sent.headers.get('Content-Type'), 'text/html'); - }); - - it('sends da-admin the document as a data form part', async () => { - const sent = await written(legacy, '

hi

'); - - const form = await sent.formData(); - - assert.strictEqual(await form.get('data').text(), '

hi

'); - assert.strictEqual(form.get('data').type, 'text/html'); - }); - - it('never wraps a source-bus write in a multipart envelope', async () => { - const body = await (await written(bus)).text(); - - assert.ok(!body.includes('Content-Disposition'), `envelope leaked into the body: ${body}`); - assert.ok(!body.includes('form-data'), `envelope leaked into the body: ${body}`); - }); - - it('posts to the url the store resolved', async () => { - const b = await written(bus); - const l = await written(legacy); - - assert.strictEqual(b.method, 'POST'); - assert.strictEqual(b.url, 'https://api.aem.live/org/sites/site/source/doc.html'); - assert.strictEqual(l.method, 'POST'); - assert.strictEqual(l.url, 'https://admin.da.live/source/org/site/doc.html'); - }); - - it('authorizes both writes with the caller token', async () => { - assert.strictEqual((await written(bus)).headers.get('Authorization'), 'Bearer t'); - assert.strictEqual((await written(legacy)).headers.get('Authorization'), 'Bearer t'); - }); - - // neither store's writes are conditional. Only the source bus sets an etag on a read, and - // nothing carries it into the save: a marker on the connection uri would be minted once per - // page load while UE saves many times against it, so no version pin could stay fresh. - it('sends no precondition to either store', async () => { - const sent = [await written(bus), await written(legacy)]; - - sent.forEach(({ headers }) => { - assert.strictEqual(headers.get('If-Match'), null); - assert.strictEqual(headers.get('If-None-Match'), null); - assert.strictEqual(headers.get('If-Unmodified-Since'), null); - }); - }); - }); }); From f41ce8e7aafc25d6307c6a003e3649b6557832a5 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Fri, 7 Aug 2026 10:19:15 +0200 Subject: [PATCH 44/48] fix: refuse the read when /ping cannot say which store holds the site a probe that throws or times out answers undefined rather than legacy, so reads and writes answer 503 instead of picking a store on a coin flip. an answer without the header is still legacy. a rejection would have been swallowed by the allSettled race on the image path. --- src/routes/da-admin.js | 15 ++++++- src/storage/source-bus.js | 8 +++- src/utils/constants.js | 2 + test/routes/source-read.test.js | 74 ++++++++++++++++++++++++++++++-- test/routes/source-write.test.js | 32 +++++++++++++- test/storage/source-bus.test.js | 28 +++++++++--- 6 files changed, 145 insertions(+), 14 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index a8f8566b..b6d4259f 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -28,6 +28,7 @@ import { BRANCH_NOT_FOUND_HTML_MESSAGE, DEFAULT_HTML_TEMPLATE, SOURCE_BUS_READ_ONLY_MESSAGE, + SOURCE_UNDETERMINED_MESSAGE, SOURCE_UNREACHABLE_HTML_MESSAGE, SOURCE_UNREACHABLE_MESSAGE, UNAUTHORIZED_HTML_MESSAGE, @@ -103,7 +104,15 @@ async function reachStore(store, send) { * @returns {Promise} undefined when the store could not be reached */ async function readSource(env, daCtx, init) { - const store = getStore(env, daCtx, await isSourceBus(env, daCtx)); + const onSourceBus = await isSourceBus(env, daCtx); + // a store picked without an answer is a coin flip, and reading the wrong one serves the wrong + // document at 200 + if (onSourceBus === undefined) { + console.warn(`503 ${init.method} ${daCtx.sourcePath}, the store could not be determined`); + return undefined; + } + + const store = getStore(env, daCtx, onSourceBus); console.log(`-> ${init.method} ${store.url.toString()}`); return reachStore(store, () => store.fetch(store.url, init)); } @@ -249,6 +258,10 @@ export async function daSourcePost({ req, env, daCtx }) { // the payload is settled, so the only question left is where it goes const onSourceBus = await isSourceBus(env, daCtx); + if (onSourceBus === undefined) { + console.warn(`503 POST ${sourcePath}, the store could not be determined`); + return post503(SOURCE_UNDETERMINED_MESSAGE); + } if (onSourceBus) { console.log(`405 POST ${sourcePath}, writes to the source bus are refused through the preview proxy. write directly to the source bus instead.`); diff --git a/src/storage/source-bus.js b/src/storage/source-bus.js index c74c394c..a6096786 100644 --- a/src/storage/source-bus.js +++ b/src/storage/source-bus.js @@ -15,9 +15,13 @@ const UPGRADE_HEADER = 'x-api-upgrade-available'; /** * Asks `/ping` whether a site is on the source bus. + * + * An answer without the header is legacy: helix-admin sets it when config resolution succeeded and + * named the API. No answer at all is not, so the caller refuses rather than picking a store. + * * @param {Object} env worker env, `HLX_ADMIN` is where the probe goes * @param {Object} daCtx - * @returns {Promise} + * @returns {Promise} undefined when the probe could not answer */ export default async function isSourceBus(env, daCtx) { const { org, site } = daCtx; @@ -31,6 +35,6 @@ export default async function isSourceBus(env, daCtx) { return response.headers.get(UPGRADE_HEADER) !== null; } catch (e) { console.warn(`[source] ${key} ping failed: ${e.name}: ${e.message}`); - return false; + return undefined; } } diff --git a/src/utils/constants.js b/src/utils/constants.js index b26a5f5c..69706aed 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -55,6 +55,8 @@ export const SOURCE_UNREACHABLE_HTML_MESSAGE = '

503: Content sto export const SOURCE_UNREACHABLE_MESSAGE = 'The store that holds this document did not answer, so nothing was written. Please retry.'; +export const SOURCE_UNDETERMINED_MESSAGE = 'Which store holds this document could not be determined, so nothing was written. Please retry.'; + export const SOURCE_BUS_READ_ONLY_MESSAGE = 'This site is on the source bus, which this proxy only reads. Nothing was written, and retrying will not help.'; export const DEFAULT_UNAUTHORIZED_HTML_MESSAGE = '

401: Unauthorized

'; diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 5af043e7..72a3b75d 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -24,13 +24,13 @@ const authedReq = (url) => new Request(url, { headers: { Authorization: 'Bearer */ const build = async (overrides = {}) => { const { - onSourceBus = false, bus = () => new Response('from the source bus', { status: 200, headers: { etag: '"busetag"' } }), legacy = () => new Response('from da-admin', { status: 200 }), } = overrides; - // 'headHtml' in overrides rather than a destructured default, so passing - // `{ headHtml: undefined }` really does simulate a missing head.html + // `in overrides` rather than a destructured default on these two, so passing an explicit + // undefined really does simulate a missing head.html and a probe that could not answer const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; + const onSourceBus = 'onSourceBus' in overrides ? overrides.onSourceBus : false; const seen = { bus: [], legacy: [], ue: 0, lookups: 0, }; @@ -75,6 +75,54 @@ const build = async (overrides = {}) => { return { ...mod, env, seen }; }; +describe('when /ping cannot say which store holds the site', () => { + afterEach(() => { + delete globalThis.fetch; + }); + + // picking a store without an answer is a coin flip, and reading the wrong one hands the author + // the wrong document at 200 + it('refuses an html read with 503 and touches neither store', async () => { + const { daSourceGet, env, seen } = await build({ onSourceBus: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + it('asks the caller to retry', async () => { + const { daSourceGet, env } = await build({ onSourceBus: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.ok(Number(res.headers.get('Retry-After')) > 0); + }); + + it('refuses a non-html read too', async () => { + const { daSourceGet, env, seen } = await build({ onSourceBus: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); + + it('refuses a HEAD with 503 and no body', async () => { + const { daSourceHead, env, seen } = await build({ onSourceBus: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(await res.text(), ''); + assert.strictEqual(seen.bus.length + seen.legacy.length, 0); + }); +}); + describe('reading from the store that holds the site', () => { afterEach(() => { delete globalThis.fetch; @@ -469,6 +517,26 @@ describe('reading from the store that holds the site', () => { it(`still prefers the store at 200 on a ${handler.toUpperCase()}`, async () => { assert.strictEqual(await raced(handler, 200, 404), 200); }); + + // the reason an undetermined store answers 503 rather than throwing: allSettled turns a + // rejection into "no answer" and the proxy's 404 would win, claiming the image is absent + it(`answers 503 when the store is undetermined on a ${handler.toUpperCase()}`, async () => { + const mod = await esmock(`../../src/handlers/${handler}.js`, { + '../../src/routes/da-admin.js': { + daSourceGet: async () => { throw new TypeError('fetch failed'); }, + daSourceHead: async () => { throw new TypeError('fetch failed'); }, + }, + '../../src/routes/aem-proxy.js': { + handleAEMProxyRequest: async () => new Response('', { status: 404 }), + }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const thrown = (await mod.default({ req, env: {}, daCtx: getDaCtx(req) })).status; + + assert.strictEqual(thrown, 404, 'a rejection is swallowed and the proxy answers'); + assert.strictEqual(await raced(handler, 503, 404), 503, 'a 503 response is not'); + }); }); // getHandler races an image read against *.aem.page and takes the proxy answer whenever the diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index 90af547d..fc91c0ea 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -14,7 +14,7 @@ import assert from 'assert'; import esmock from 'esmock'; import { getDaCtx } from '../../src/utils/daCtx.js'; -import { SOURCE_BUS_READ_ONLY_MESSAGE } from '../../src/utils/constants.js'; +import { SOURCE_BUS_READ_ONLY_MESSAGE, SOURCE_UNDETERMINED_MESSAGE } from '../../src/utils/constants.js'; const AT = 'https://main--site--org.ue.da.live/folder/content'; const DOC = '

the author typed this

'; @@ -26,7 +26,11 @@ const uePost = (url, html = DOC) => { return new Request(url, { method: 'POST', body, headers: { Authorization: 'Bearer t' } }); }; -const build = async ({ onSourceBus = false, status = 201 } = {}) => { +const build = async (overrides = {}) => { + const { status = 201 } = overrides; + // `in overrides` rather than a destructured default, so passing an explicit undefined really + // does simulate a probe that could not answer + const onSourceBus = 'onSourceBus' in overrides ? overrides.onSourceBus : false; const seen = { bus: [], legacy: [], lookups: 0, order: [], }; @@ -120,6 +124,30 @@ describe('writing to the store that holds the site', () => { }); }); + // a write is the one operation a wrong store cannot be walked back from, so no answer means no + // write rather than a guess + describe('when /ping cannot say which store holds the site', () => { + it('is refused with 503 and touches neither store', async () => { + const { res, seen } = await post({ onSourceBus: undefined }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(seen.bus.length, 0); + assert.strictEqual(seen.legacy.length, 0); + }); + + it('asks the caller to retry, unlike the source-bus refusal', async () => { + const { res } = await post({ onSourceBus: undefined }); + + assert.ok(Number(res.headers.get('Retry-After')) > 0); + }); + + it('says which of the two refusals it is', async () => { + const { res } = await post({ onSourceBus: undefined }); + + assert.strictEqual(await res.text(), SOURCE_UNDETERMINED_MESSAGE); + }); + }); + describe('a legacy site', () => { it('writes to da-admin over the service binding', async () => { const { seen } = await post({}); diff --git a/test/storage/source-bus.test.js b/test/storage/source-bus.test.js index 17c3691f..26320c50 100644 --- a/test/storage/source-bus.test.js +++ b/test/storage/source-bus.test.js @@ -126,23 +126,39 @@ describe('isSourceBus', () => { assert.strictEqual(await isSourceBus(env, daCtx()), false); }); }); + }); - it('answers false when the probe throws', async () => { + // an answer without the header is legacy. no answer is not an answer, and the caller refuses + // rather than picking a store on a coin flip + describe('when /ping cannot answer', () => { + it('answers undefined when the probe throws', async () => { stubFetch(() => { throw new TypeError('fetch failed'); }); - assert.strictEqual(await isSourceBus(env, daCtx()), false); + assert.strictEqual(await isSourceBus(env, daCtx()), undefined); }); - // HLX_ADMIN is a wrangler.toml constant, so this is a broken deploy rather than a runtime - // condition. It answers legacy instead of throwing out of every read. - it('answers false without asking when HLX_ADMIN is unusable', async () => { + it('answers undefined when HLX_ADMIN is unusable, without asking', async () => { stubFetch(upgraded); - assert.strictEqual(await isSourceBus({ AEM_API: 'https://api.aem.live' }, daCtx()), false); + assert.strictEqual( + await isSourceBus({ AEM_API: 'https://api.aem.live' }, daCtx()), + undefined, + ); assert.strictEqual(calls.length, 0); }); + + // the distinction the caller acts on: false is a store, undefined is no store + it('is distinguishable from a legacy answer', async () => { + stubFetch(() => ping()); + assert.strictEqual(await isSourceBus(env, daCtx()), false); + + stubFetch(() => { + throw new TypeError('fetch failed'); + }); + assert.strictEqual(await isSourceBus(env, daCtx()), undefined); + }); }); describe('when there is no site to ask about', () => { From 9119b98120d422d8abe87342ce6c76f68084f405 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 10 Aug 2026 08:53:53 +0200 Subject: [PATCH 45/48] test: x-error on the 503s says why the store did not answer --- test/routes/source-read.test.js | 83 ++++++++++++++++++++++++++++++++ test/routes/source-write.test.js | 20 ++++++++ 2 files changed, 103 insertions(+) diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 72a3b75d..28febbee 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -121,6 +121,26 @@ describe('when /ping cannot say which store holds the site', () => { assert.strictEqual(await res.text(), ''); assert.strictEqual(seen.bus.length + seen.legacy.length, 0); }); + + // both 503s carry the same status and a body nobody parses, so the header is what tells an + // unanswerable probe apart from a store that was picked and then failed + it('names the failed probe in x-error', async () => { + const { daSourceGet, env } = await build({ onSourceBus: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.match(res.headers.get('x-error'), /ping/); + }); + + it('names the failed probe on a HEAD too', async () => { + const { daSourceHead, env } = await build({ onSourceBus: undefined }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.match(res.headers.get('x-error'), /ping/); + }); }); describe('reading from the store that holds the site', () => { @@ -267,6 +287,69 @@ describe('reading from the store that holds the site', () => { assert.strictEqual(await res.text(), ''); assert.ok(Number(res.headers.get('Retry-After')) > 0); }); + + // a rate limit, a DNS failure and a timeout all arrive here as the same 503, and the worker + // log is not where the caller is looking + it('names the cause in x-error on an html read', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('Network connection lost'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TypeError: Network connection lost'); + }); + + it('names the cause on a non-html read', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('Network connection lost'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/photo.png'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TypeError: Network connection lost'); + }); + + it('names the cause on a HEAD', async () => { + const { daSourceHead, env } = await build({ + onSourceBus: true, + bus: () => { throw new DOMException('The operation timed out', 'TimeoutError'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceHead({ env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TimeoutError: The operation timed out'); + }); + + it('names the cause when da-admin is unreachable', async () => { + const { daSourceGet, env } = await build({ + legacy: () => { throw new TypeError('Network connection lost'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TypeError: Network connection lost'); + }); + + // a header value cannot span lines, so a message carrying a stack would throw where the 503 + // is built and turn the 503 into a 500 + it('collapses a multi-line cause onto one line', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('lost\n at fetch'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TypeError: lost at fetch'); + }); }); describe('what a store status means', () => { diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index fc91c0ea..a0942428 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -146,6 +146,12 @@ describe('writing to the store that holds the site', () => { assert.strictEqual(await res.text(), SOURCE_UNDETERMINED_MESSAGE); }); + + it('names the failed probe in x-error', async () => { + const { res } = await post({ onSourceBus: undefined }); + + assert.match(res.headers.get('x-error'), /ping/); + }); }); describe('a legacy site', () => { @@ -257,6 +263,20 @@ describe('writing to the store that holds the site', () => { assert.strictEqual(res.status, 503); }); + + // a save that failed on a rate limit and one that failed on a dropped connection are the same + // 503 to the editor, and only one of them is worth retrying at once + it('names the cause in x-error', async () => { + const { daSourcePost, env } = await build({}); + env.daadmin.fetch = async () => { + throw new TypeError('Network connection lost'); + }; + const req = uePost(AT); + + const res = await daSourcePost({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TypeError: Network connection lost'); + }); }); describe('a non-html path', () => { From 75fd95b8045241ce4158822336da53d86461bc74 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 10 Aug 2026 08:53:53 +0200 Subject: [PATCH 46/48] fix: carry the store failure cause on the 503s as x-error --- src/handlers/get.js | 2 -- src/responses/index.js | 20 ++++++++++++++------ src/routes/da-admin.js | 42 +++++++++++++++++++++++++----------------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/handlers/get.js b/src/handlers/get.js index 4029bcb2..591cada5 100644 --- a/src/handlers/get.js +++ b/src/handlers/get.js @@ -44,8 +44,6 @@ export default async function getHandler({ req, env, daCtx }) { } else if (aemRes?.status === 200) { response = aemRes; } else if (storeRes && storeRes.status >= 500) { - // the store could not answer, so neither can we. Taking the proxy's 404 here would say the - // image does not exist when the truth is that we could not find out. response = storeRes; } else if (aemRes) { response = aemRes; diff --git a/src/responses/index.js b/src/responses/index.js index 499a5a52..41758933 100644 --- a/src/responses/index.js +++ b/src/responses/index.js @@ -13,6 +13,14 @@ import { DEFAULT_UNAUTHORIZED_HTML_MESSAGE } from '../utils/constants.js'; const RETRY_AFTER_SECONDS = '5'; +// a 503 says the store did not answer and the body says it again. Only `x-error` says which of a +// rate limit, a timeout or a dropped connection it was, without reading the worker log. +function retryHeaders(error) { + const headers = [['Retry-After', RETRY_AFTER_SECONDS]]; + if (error) headers.push(['x-error', error]); + return headers; +} + export function daResp({ body, status, contentType, contentLength, headers: extraHeaders, }) { @@ -49,23 +57,23 @@ export function get415(message = '') { return daResp({ body: message, status: 415, contentType: 'text/html' }); } -export function get503(message = '') { +export function get503(message = '', error = '') { return daResp({ body: message, status: 503, contentType: 'text/html', - headers: [['Retry-After', RETRY_AFTER_SECONDS]], + headers: retryHeaders(error), }); } // a refused write is never rendered. The Universal Editor Service embeds the body verbatim in // its problem+json error string, so plain text is what an author is shown. -export function post503(message = '') { +export function post503(message = '', error = '') { return daResp({ body: message, status: 503, contentType: 'text/plain; charset=utf-8', - headers: [['Retry-After', RETRY_AFTER_SECONDS]], + headers: retryHeaders(error), }); } @@ -83,8 +91,8 @@ export function head401() { return new Response(null, { status: 401 }); } -export function head503() { - return new Response(null, { status: 503, headers: { 'Retry-After': RETRY_AFTER_SECONDS } }); +export function head503(error = '') { + return new Response(null, { status: 503, headers: retryHeaders(error) }); } export function head404() { diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index b6d4259f..e9a3fb9b 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -40,6 +40,8 @@ import { restoreAbsoluteImages } from '../render/rewrite-images.js'; const HTML_POST_TYPE = 'text/html'; +const PROBE_FAILED = 'the /ping probe failed, so which store holds this site is unknown'; + export function isHtmlPostType(type) { if (!type) return true; return type.split(';')[0].trim().toLowerCase() === HTML_POST_TYPE; @@ -87,29 +89,35 @@ async function getPageTemplate(env, daCtx, aemCtx) { } /** - * Sends a request to a store and answers 503 when it could not be reached at all. + * Sends a request to a store and reports why it could not be reached at all. + * + * The cause is collapsed onto one line because it is answered as a header, which cannot span + * lines, and a `TypeError` carrying a stack would otherwise throw where the 503 is built. + * + * @returns {Promise<{response?: Response, error?: string}>} */ async function reachStore(store, send) { try { - return await send(); + return { response: await send() }; } catch (e) { - console.warn(`503 ${store.url}, the store could not be reached: ${e.name}: ${e.message}`); - return undefined; + const error = `${e.name}: ${e.message}`.replace(/\s+/g, ' ').trim(); + console.warn(`503 ${store.url}, the store could not be reached: ${error}`); + return { error }; } } /** * Reads from the store that holds the site. * - * @returns {Promise} undefined when the store could not be reached + * @returns {Promise<{response?: Response, error?: string}>} `error` says why there is no response */ async function readSource(env, daCtx, init) { const onSourceBus = await isSourceBus(env, daCtx); // a store picked without an answer is a coin flip, and reading the wrong one serves the wrong // document at 200 if (onSourceBus === undefined) { - console.warn(`503 ${init.method} ${daCtx.sourcePath}, the store could not be determined`); - return undefined; + console.warn(`503 ${init.method} ${daCtx.sourcePath}, ${PROBE_FAILED}`); + return { error: PROBE_FAILED }; } const store = getStore(env, daCtx, onSourceBus); @@ -139,15 +147,15 @@ export async function daSourceGet({ req, env, daCtx }) { if (ext !== 'html') { // for non-HTML files, simply proxy the request without processing. A refusal is passed on as // itself: nothing renders an image, so the da:401 shell would only corrupt it. - const response = await readSource(env, daCtx, { method: 'GET', headers }); - if (!response) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); + const { response, error } = await readSource(env, daCtx, { method: 'GET', headers }); + if (!response) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE, error); console.log(`<- ${daCtx.sourcePath}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } // the store lookup costs a round trip, so it runs alongside head.html rather than after it const aemCtx = getAemCtx(env, daCtx); - const [headHtml, sourceResp] = await Promise.all([ + const [headHtml, { response: sourceResp, error: sourceError }] = await Promise.all([ getAEMHtml(aemCtx, '/head.html'), readSource(env, daCtx, { method: 'GET', headers }), ]); @@ -159,7 +167,7 @@ export async function daSourceGet({ req, env, daCtx }) { } return get404(BRANCH_NOT_FOUND_HTML_MESSAGE); } - if (!sourceResp) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE); + if (!sourceResp) return get503(SOURCE_UNREACHABLE_HTML_MESSAGE, sourceError); console.log(`<- ${daCtx.sourcePath}. ${sourceResp.status} ${sourceResp.statusText}`, { status: sourceResp.status, statusText: sourceResp.statusText }); // the store is the only thing to see the token, and the authorbus extension recovers off the @@ -216,8 +224,8 @@ export async function daSourceHead({ env, daCtx }) { const headers = new Headers(); headers.set('Authorization', authToken); - const response = await readSource(env, daCtx, { method: 'HEAD', headers }); - if (!response) return head503(); + const { response, error } = await readSource(env, daCtx, { method: 'HEAD', headers }); + if (!response) return head503(error); console.log(`<- HEAD ${daCtx.sourcePath}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return new Response(null, { status: response.status, headers: response.headers }); } @@ -259,8 +267,8 @@ export async function daSourcePost({ req, env, daCtx }) { // the payload is settled, so the only question left is where it goes const onSourceBus = await isSourceBus(env, daCtx); if (onSourceBus === undefined) { - console.warn(`503 POST ${sourcePath}, the store could not be determined`); - return post503(SOURCE_UNDETERMINED_MESSAGE); + console.warn(`503 POST ${sourcePath}, ${PROBE_FAILED}`); + return post503(SOURCE_UNDETERMINED_MESSAGE, PROBE_FAILED); } if (onSourceBus) { @@ -273,12 +281,12 @@ export async function daSourcePost({ req, env, daCtx }) { const body = new FormData(); body.set('data', new Blob([bodyContent], { type: 'text/html' })); console.log(`-> ${store.url.toString()}`); - const response = await reachStore(store, () => store.fetch(new Request(store.url, { + const { response, error } = await reachStore(store, () => store.fetch(new Request(store.url, { method: 'POST', body, headers: { Authorization: authToken }, }))); - if (!response) return post503(SOURCE_UNREACHABLE_MESSAGE); + if (!response) return post503(SOURCE_UNREACHABLE_MESSAGE, error); console.log(`<- ${store.url.toString()}. ${response.status} ${response.statusText}`, { status: response.status, statusText: response.statusText }); return response; } From 82edec10b721b1872c24d3fb3e7d507bc6a54282 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 10 Aug 2026 10:16:48 +0200 Subject: [PATCH 47/48] test: the /ping probe reports its failure by throwing --- test/routes/source-read.test.js | 40 ++++++++++++++++++++++++++++++++ test/routes/source-write.test.js | 12 ++++++++++ test/storage/source-bus.test.js | 17 +++++++------- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/test/routes/source-read.test.js b/test/routes/source-read.test.js index 28febbee..d51cb27b 100644 --- a/test/routes/source-read.test.js +++ b/test/routes/source-read.test.js @@ -31,6 +31,7 @@ const build = async (overrides = {}) => { // undefined really does simulate a missing head.html and a probe that could not answer const headHtml = 'headHtml' in overrides ? overrides.headHtml : ''; const onSourceBus = 'onSourceBus' in overrides ? overrides.onSourceBus : false; + const probeError = 'probeError' in overrides ? overrides.probeError : new TypeError('fetch failed'); const seen = { bus: [], legacy: [], ue: 0, lookups: 0, }; @@ -54,6 +55,8 @@ const build = async (overrides = {}) => { '../../src/storage/source-bus.js': { default: async () => { seen.lookups += 1; + // the probe reports a failure by throwing, so undefined stands for "could not answer" + if (onSourceBus === undefined) throw probeError; return onSourceBus; }, }, @@ -141,6 +144,43 @@ describe('when /ping cannot say which store holds the site', () => { assert.match(res.headers.get('x-error'), /ping/); }); + + // a read answers the same 503 and the same body whichever of the two failed, so the header is + // the only thing on the wire that separates a timeout from a dropped connection + it('carries the probe cause, not a category', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: undefined, + probeError: new DOMException('timed out', 'TimeoutError'), + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), '/ping failed: TimeoutError: timed out'); + }); + + // rendering a thrown non-Error as "undefined: undefined" would leave the 503 saying nothing + it('survives a thrown non-Error', async () => { + const { daSourceGet, env } = await build({ onSourceBus: undefined, probeError: 'boom' }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.status, 503); + assert.strictEqual(res.headers.get('x-error'), '/ping failed: Error: boom'); + }); + + it('tells a probe failure apart from a store failure', async () => { + const { daSourceGet, env } = await build({ + onSourceBus: true, + bus: () => { throw new TypeError('fetch failed'); }, + }); + const req = authedReq('https://main--site--org.ue.da.live/folder/content'); + + const res = await daSourceGet({ req, env, daCtx: getDaCtx(req) }); + + assert.strictEqual(res.headers.get('x-error'), 'TypeError: fetch failed'); + }); }); describe('reading from the store that holds the site', () => { diff --git a/test/routes/source-write.test.js b/test/routes/source-write.test.js index a0942428..ec9ce54f 100644 --- a/test/routes/source-write.test.js +++ b/test/routes/source-write.test.js @@ -31,6 +31,7 @@ const build = async (overrides = {}) => { // `in overrides` rather than a destructured default, so passing an explicit undefined really // does simulate a probe that could not answer const onSourceBus = 'onSourceBus' in overrides ? overrides.onSourceBus : false; + const probeError = 'probeError' in overrides ? overrides.probeError : new TypeError('fetch failed'); const seen = { bus: [], legacy: [], lookups: 0, order: [], }; @@ -72,6 +73,8 @@ const build = async (overrides = {}) => { default: async () => { seen.lookups += 1; seen.order.push('lookup'); + // the probe reports a failure by throwing, so undefined stands for "could not answer" + if (onSourceBus === undefined) throw probeError; return onSourceBus; }, }, @@ -152,6 +155,15 @@ describe('writing to the store that holds the site', () => { assert.match(res.headers.get('x-error'), /ping/); }); + + it('carries the probe cause, not a category', async () => { + const { res } = await post({ + onSourceBus: undefined, + probeError: new DOMException('timed out', 'TimeoutError'), + }); + + assert.strictEqual(res.headers.get('x-error'), '/ping failed: TimeoutError: timed out'); + }); }); describe('a legacy site', () => { diff --git a/test/storage/source-bus.test.js b/test/storage/source-bus.test.js index 26320c50..8093d3fd 100644 --- a/test/storage/source-bus.test.js +++ b/test/storage/source-bus.test.js @@ -131,25 +131,24 @@ describe('isSourceBus', () => { // an answer without the header is legacy. no answer is not an answer, and the caller refuses // rather than picking a store on a coin flip describe('when /ping cannot answer', () => { - it('answers undefined when the probe throws', async () => { + // the cause reaches the caller, which reports it on the 503 as `x-error`. swallowing it here + // would leave a timeout and a dropped connection indistinguishable + it('lets the failure through', async () => { stubFetch(() => { throw new TypeError('fetch failed'); }); - assert.strictEqual(await isSourceBus(env, daCtx()), undefined); + await assert.rejects(isSourceBus(env, daCtx()), { message: 'fetch failed' }); }); - it('answers undefined when HLX_ADMIN is unusable, without asking', async () => { + it('lets it through when HLX_ADMIN is unusable, without asking', async () => { stubFetch(upgraded); - assert.strictEqual( - await isSourceBus({ AEM_API: 'https://api.aem.live' }, daCtx()), - undefined, - ); + await assert.rejects(isSourceBus({ AEM_API: 'https://api.aem.live' }, daCtx())); assert.strictEqual(calls.length, 0); }); - // the distinction the caller acts on: false is a store, undefined is no store + // the distinction the caller acts on: false is a store, a failure is no store it('is distinguishable from a legacy answer', async () => { stubFetch(() => ping()); assert.strictEqual(await isSourceBus(env, daCtx()), false); @@ -157,7 +156,7 @@ describe('isSourceBus', () => { stubFetch(() => { throw new TypeError('fetch failed'); }); - assert.strictEqual(await isSourceBus(env, daCtx()), undefined); + await assert.rejects(isSourceBus(env, daCtx())); }); }); From b522480751337b7ccf551ed47e50d00ab72b4297 Mon Sep 17 00:00:00 2001 From: Ben Peter Date: Mon, 10 Aug 2026 10:16:48 +0200 Subject: [PATCH 48/48] fix: put the real /ping cause in x-error, not a category --- src/routes/da-admin.js | 43 +++++++++++++++++++++++++-------------- src/storage/source-bus.js | 17 ++++++---------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/routes/da-admin.js b/src/routes/da-admin.js index e9a3fb9b..d2484b6f 100644 --- a/src/routes/da-admin.js +++ b/src/routes/da-admin.js @@ -40,7 +40,22 @@ import { restoreAbsoluteImages } from '../render/rewrite-images.js'; const HTML_POST_TYPE = 'text/html'; -const PROBE_FAILED = 'the /ping probe failed, so which store holds this site is unknown'; +/** + * Renders a failure for the `x-error` header. + */ +function causeOf(e) { + return `${e?.name ?? 'Error'}: ${e?.message ?? e}` + .replace(/[^\x20-\x7e]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 1024); +} + +function probeFailed(e, method, sourcePath) { + const cause = `/ping failed: ${causeOf(e)}`; + console.warn(`503 ${method} ${sourcePath}, ${cause}`); + return cause; +} export function isHtmlPostType(type) { if (!type) return true; @@ -91,16 +106,13 @@ async function getPageTemplate(env, daCtx, aemCtx) { /** * Sends a request to a store and reports why it could not be reached at all. * - * The cause is collapsed onto one line because it is answered as a header, which cannot span - * lines, and a `TypeError` carrying a stack would otherwise throw where the 503 is built. - * * @returns {Promise<{response?: Response, error?: string}>} */ async function reachStore(store, send) { try { return { response: await send() }; } catch (e) { - const error = `${e.name}: ${e.message}`.replace(/\s+/g, ' ').trim(); + const error = causeOf(e); console.warn(`503 ${store.url}, the store could not be reached: ${error}`); return { error }; } @@ -112,12 +124,11 @@ async function reachStore(store, send) { * @returns {Promise<{response?: Response, error?: string}>} `error` says why there is no response */ async function readSource(env, daCtx, init) { - const onSourceBus = await isSourceBus(env, daCtx); - // a store picked without an answer is a coin flip, and reading the wrong one serves the wrong - // document at 200 - if (onSourceBus === undefined) { - console.warn(`503 ${init.method} ${daCtx.sourcePath}, ${PROBE_FAILED}`); - return { error: PROBE_FAILED }; + let onSourceBus; + try { + onSourceBus = await isSourceBus(env, daCtx); + } catch (e) { + return { error: probeFailed(e, init.method, daCtx.sourcePath) }; } const store = getStore(env, daCtx, onSourceBus); @@ -265,10 +276,12 @@ export async function daSourcePost({ req, env, daCtx }) { const bodyContent = toHtml(bodyNode); // the payload is settled, so the only question left is where it goes - const onSourceBus = await isSourceBus(env, daCtx); - if (onSourceBus === undefined) { - console.warn(`503 POST ${sourcePath}, ${PROBE_FAILED}`); - return post503(SOURCE_UNDETERMINED_MESSAGE, PROBE_FAILED); + let onSourceBus; + try { + onSourceBus = await isSourceBus(env, daCtx); + } catch (e) { + const cause = probeFailed(e, 'POST', sourcePath); + return post503(SOURCE_UNDETERMINED_MESSAGE, cause); } if (onSourceBus) { diff --git a/src/storage/source-bus.js b/src/storage/source-bus.js index a6096786..9cfe6b89 100644 --- a/src/storage/source-bus.js +++ b/src/storage/source-bus.js @@ -17,24 +17,19 @@ const UPGRADE_HEADER = 'x-api-upgrade-available'; * Asks `/ping` whether a site is on the source bus. * * An answer without the header is legacy: helix-admin sets it when config resolution succeeded and - * named the API. No answer at all is not, so the caller refuses rather than picking a store. + * named the API. A probe that cannot answer throws, so the caller refuses with the cause rather + * than picking a store. * * @param {Object} env worker env, `HLX_ADMIN` is where the probe goes * @param {Object} daCtx - * @returns {Promise} undefined when the probe could not answer + * @returns {Promise} */ export default async function isSourceBus(env, daCtx) { const { org, site } = daCtx; // an unparseable hostname leaves org and site undefined, and there is no site to ask about if (!org || !site) return false; - const key = `${org}/${site}`; - try { - const url = new URL(`/ping/${key}`, env.HLX_ADMIN); - const response = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }); - return response.headers.get(UPGRADE_HEADER) !== null; - } catch (e) { - console.warn(`[source] ${key} ping failed: ${e.name}: ${e.message}`); - return undefined; - } + const url = new URL(`/ping/${org}/${site}`, env.HLX_ADMIN); + const response = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }); + return response.headers.get(UPGRADE_HEADER) !== null; }