From ac93025fd112fdd70c376b3ad05f8d95c2154d68 Mon Sep 17 00:00:00 2001 From: Ramachandra Avuthu Date: Wed, 15 Jul 2026 15:51:14 -0400 Subject: [PATCH 1/2] localize the inherited content --- nx/blocks/loc/utils/msm.js | 63 ++++++++++++++++++++++++ nx/blocks/loc/views/translate/index.js | 6 +-- nx/blocks/loc/views/validate/validate.js | 14 ++++-- test/utils/tree.test.js | 8 ++- 4 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 nx/blocks/loc/utils/msm.js diff --git a/nx/blocks/loc/utils/msm.js b/nx/blocks/loc/utils/msm.js new file mode 100644 index 000000000..6252fc38d --- /dev/null +++ b/nx/blocks/loc/utils/msm.js @@ -0,0 +1,63 @@ +import { DA_ADMIN } from '../../../../nx2/utils/utils.js'; +import { daFetch } from '../../../../nx2/utils/api.js'; + +const sourceOf = (row) => row.base || row.source; +const linkedOf = (row) => row.satellite || row.linked; + +const msmRowsCache = {}; + +export function fetchMsmRows(org) { + msmRowsCache[org] ??= (async () => { + try { + const resp = await daFetch({ url: `${DA_ADMIN}/config/${org}/` }); + if (resp.status === 404) return []; + if (!resp.ok) throw new Error(`config ${resp.status}`); + const json = await resp.json(); + return json?.msm?.data || []; + } catch { + delete msmRowsCache[org]; + return []; + } + })(); + return msmRowsCache[org]; +} + +export function getSourceChain(rows, site) { + const chain = []; + const visited = new Set(); + let current = site; + while (current && !visited.has(current)) { + visited.add(current); + const linkedSite = current; + const parentRow = rows.find((row) => linkedOf(row) === linkedSite); + const parent = parentRow && sourceOf(parentRow); + if (!parent) break; + chain.push(parent); + current = parent; + } + return chain; +} + +export async function fetchWithMsmFallback({ org, site, daPath, opts }) { + const fetchFrom = (fromSite) => daFetch({ + url: `${DA_ADMIN}/source/${org}/${fromSite}${daPath}`, + opts, + }); + + const selfResp = await fetchFrom(site); + if (selfResp.ok) return { resp: selfResp, resolvedSite: site, inherited: false }; + + if (selfResp.status !== 404) return { resp: selfResp, resolvedSite: site, inherited: false }; + + const rows = await fetchMsmRows(org); + if (!rows.length) return { resp: selfResp, resolvedSite: site, inherited: false }; + + const chain = getSourceChain(rows, site); + for (const ancestor of chain) { + // eslint-disable-next-line no-await-in-loop + const resp = await fetchFrom(ancestor); + if (resp.ok) return { resp, resolvedSite: ancestor, inherited: true }; + } + + return { resp: selfResp, resolvedSite: site, inherited: false }; +} diff --git a/nx/blocks/loc/views/translate/index.js b/nx/blocks/loc/views/translate/index.js index 3d44665ef..562bb2c45 100644 --- a/nx/blocks/loc/views/translate/index.js +++ b/nx/blocks/loc/views/translate/index.js @@ -1,8 +1,7 @@ -import { DA_ADMIN } from '../../../../../nx2/utils/utils.js'; import { Queue } from '../../../../../nx2/public/utils/tree.js'; -import { daFetch } from '../../../../../nx2/utils/api.js'; import { convertPath, createSnapshotPrefix, fetchConfig } from '../../utils/utils.js'; +import { fetchWithMsmFallback } from '../../utils/msm.js'; import { MAX_CONCURRENT_READS, MAX_CONCURRENT_WRITES, mergeCopy, overwriteCopy } from '../../project/index.js'; let CONNECTOR; @@ -50,7 +49,8 @@ export async function getUrls( // Fetch the content and add DNT const fetchUrl = async (url) => { const opts = { headers: { 'Cache-Control': 'no-cache' } }; - const resp = await daFetch({ url: `${DA_ADMIN}/source/${org}/${site}${url.daDestPath}`, opts }); + // Resolve MSM inheritance so inherited pages translate from their source site. + const { resp } = await fetchWithMsmFallback({ org, site, daPath: url.daDestPath, opts }); if (!resp.ok) { url.error = `Error fetching content from ${url.daDestPath} - ${resp.status}`; return; diff --git a/nx/blocks/loc/views/validate/validate.js b/nx/blocks/loc/views/validate/validate.js index a6f7754d5..0ed51c9c8 100644 --- a/nx/blocks/loc/views/validate/validate.js +++ b/nx/blocks/loc/views/validate/validate.js @@ -1,10 +1,9 @@ import { LitElement, html, nothing } from 'da-lit'; -import { DA_ADMIN } from '../../../../../nx2/utils/utils.js'; import { loadStyle } from '../../../../../nx2/scripts/nx.js'; -import { daFetch } from '../../../../../nx2/utils/api.js'; import { Queue } from '../../../../../nx2/public/utils/tree.js'; import { convertPath, createSnapshotPrefix, fetchConfig } from '../../utils/utils.js'; +import { fetchWithMsmFallback } from '../../utils/msm.js'; import { getFragmentUrls } from './validate-utils.js'; import { MAX_CONCURRENT_READS } from '../../project/index.js'; @@ -101,16 +100,21 @@ class NxLocValidate extends LitElement { const isSheet = pathname.endsWith('.json'); const extPath = isSheet ? pathname : `${pathname}.html`; const snapshotUrlFragment = createSnapshotPrefix(this._snapshot); - const daUrl = `${DA_ADMIN}/source/${this._org}/${this._site}${snapshotUrlFragment}${extPath}`; - const resp = await daFetch({ url: daUrl }); + const daPath = `${snapshotUrlFragment}${extPath}`; + const { resp, resolvedSite, inherited } = await fetchWithMsmFallback({ + org: this._org, + site: this._site, + daPath, + }); const text = await resp.text(); const ok = resp.status === 200; url.status = ok ? 'ready' : 'error - not found'; url.checked = ok; + url.inherited = ok && inherited; url.sheet = isSheet; url.extPath = extPath; url.fragment = url.pathname.includes('/fragments/'); - url.daEdit = `${DA_LIVE}/edit#/${this._org}/${this._site}${snapshotUrlFragment}${url.pathname}`; + url.daEdit = `${DA_LIVE}/edit#/${this._org}/${resolvedSite}${snapshotUrlFragment}${url.pathname}`; if (ok) await this.findFragments(text); this.requestUpdate(); } diff --git a/test/utils/tree.test.js b/test/utils/tree.test.js index 071fcd063..529f2dac7 100644 --- a/test/utils/tree.test.js +++ b/test/utils/tree.test.js @@ -196,7 +196,9 @@ describe('Queue', () => { await queue.push('item2'); const timeDiff = timestamps[1] - timestamps[0]; - expect(timeDiff).to.be.at.least(100); + // Allow a small tolerance: browsers coarsen Date.now() (timer precision + // mitigations), so a real 100ms delay can read back a few ms short. + expect(timeDiff).to.be.at.least(95); }); it('Processes items in FIFO order', async () => { @@ -575,7 +577,9 @@ describe('crawl', () => { await results; const timeDiff = secondFetchTime - firstFetchTime; - expect(timeDiff).to.be.at.least(50); + // Allow a small tolerance: browsers coarsen Date.now() (timer precision + // mitigations), so a real 50ms delay can read back a few ms short. + expect(timeDiff).to.be.at.least(45); }); it('Resolves results promise with all files', async () => { From 5bbc76b7ba914668370b411eb1dd86ff2db1abe5 Mon Sep 17 00:00:00 2001 From: Ramachandra Avuthu Date: Thu, 16 Jul 2026 15:39:33 -0400 Subject: [PATCH 2/2] msm unit tests --- test/loc/msm.test.js | 202 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 test/loc/msm.test.js diff --git a/test/loc/msm.test.js b/test/loc/msm.test.js new file mode 100644 index 000000000..1b17fcd28 --- /dev/null +++ b/test/loc/msm.test.js @@ -0,0 +1,202 @@ +import { expect } from '@esm-bundle/chai'; +import sinon from 'sinon'; +import { DA_ADMIN } from '../../nx2/utils/utils.js'; +import { fetchMsmRows, getSourceChain, fetchWithMsmFallback } from '../../nx/blocks/loc/utils/msm.js'; + +const DA_PATH = '/en/about.html'; + +function makeResp(status, body = '') { + return { + ok: status >= 200 && status < 300, + status, + headers: new Headers(), + json: async () => body, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + }; +} + +function stubFetch({ org, config, sources = {} }) { + globalThis.fetch = sinon.stub().callsFake((url) => { + if (url.includes('/config/')) { + if (typeof config === 'number') return Promise.resolve(makeResp(config)); + return Promise.resolve(makeResp(200, { msm: { data: config || [] } })); + } + const site = Object.keys(sources).find((s) => url.includes(`/source/${org}/${s}/`)); + const status = site ? sources[site] : 404; + return Promise.resolve(makeResp(status, status === 200 ? '
content
' : '')); + }); +} + +const configCalls = () => globalThis.fetch.args.filter(([url]) => url.includes('/config/')); + +describe('msm', () => { + let originalFetch; + + beforeEach(() => { + originalFetch = globalThis.fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + sinon.restore(); + }); + + describe('getSourceChain', () => { + it('returns an empty chain when the linked site has no source', () => { + const rows = [{ source: 'source-site', linked: 'linked-site' }]; + expect(getSourceChain(rows, 'unrelated-site')).to.deep.equal([]); + }); + + it('resolves the source of a linked site', () => { + const rows = [{ source: 'source-site', linked: 'linked-site' }]; + expect(getSourceChain(rows, 'linked-site')).to.deep.equal(['source-site']); + }); + + it('walks a multi-level chain nearest-source-first', () => { + const rows = [ + { source: 'source-site', linked: 'mid-site' }, + { source: 'mid-site', linked: 'linked-site' }, + ]; + expect(getSourceChain(rows, 'linked-site')).to.deep.equal(['mid-site', 'source-site']); + }); + + it('guards against cycles without looping forever', () => { + const rows = [ + { source: 'site-a', linked: 'site-b' }, + { source: 'site-b', linked: 'site-a' }, + ]; + expect(getSourceChain(rows, 'site-a')).to.deep.equal(['site-b', 'site-a']); + }); + + it('supports the legacy base/satellite column names', () => { + const rows = [{ base: 'source-site', satellite: 'linked-site' }]; + expect(getSourceChain(rows, 'linked-site')).to.deep.equal(['source-site']); + }); + + it('falls back to the alternate column when a cell is blank', () => { + const rows = [{ base: '', source: 'source-site', satellite: '', linked: 'linked-site' }]; + expect(getSourceChain(rows, 'linked-site')).to.deep.equal(['source-site']); + }); + }); + + describe('fetchMsmRows', () => { + it('returns the msm sheet rows on a successful config fetch', async () => { + const rows = [{ source: 'source-site', linked: 'linked-site' }]; + stubFetch({ org: 'rows-ok', config: rows }); + expect(await fetchMsmRows('rows-ok')).to.deep.equal(rows); + }); + + it('returns [] when the config has no msm sheet', async () => { + globalThis.fetch = sinon.stub().resolves(makeResp(200, { other: { data: [] } })); + expect(await fetchMsmRows('rows-nomsm')).to.deep.equal([]); + }); + + it('memoizes a successful lookup (one config fetch for repeat calls)', async () => { + stubFetch({ org: 'rows-memo', config: [{ source: 'source-site', linked: 'linked-site' }] }); + await fetchMsmRows('rows-memo'); + await fetchMsmRows('rows-memo'); + expect(configCalls().length).to.equal(1); + }); + + it('caches an absent config (404) so non-MSM orgs are not re-probed', async () => { + stubFetch({ org: 'rows-404', config: 404 }); + expect(await fetchMsmRows('rows-404')).to.deep.equal([]); + await fetchMsmRows('rows-404'); + expect(configCalls().length).to.equal(1); + }); + + it('does not cache a transient failure (500) so a later call retries', async () => { + stubFetch({ org: 'rows-500', config: 500 }); + expect(await fetchMsmRows('rows-500')).to.deep.equal([]); + await fetchMsmRows('rows-500'); + expect(configCalls().length).to.equal(2); + }); + }); + + describe('fetchWithMsmFallback', () => { + it('returns the linked-site response without consulting MSM when the file exists', async () => { + stubFetch({ org: 'self-ok', sources: { 'linked-site': 200 } }); + const result = await fetchWithMsmFallback({ org: 'self-ok', site: 'linked-site', daPath: DA_PATH }); + expect(result.resp.ok).to.equal(true); + expect(result.resolvedSite).to.equal('linked-site'); + expect(result.inherited).to.equal(false); + expect(configCalls().length).to.equal(0); + }); + + it('resolves an inherited page from its source site', async () => { + stubFetch({ + org: 'inherit', + config: [{ source: 'source-site', linked: 'linked-site' }], + sources: { 'source-site': 200 }, + }); + const result = await fetchWithMsmFallback({ org: 'inherit', site: 'linked-site', daPath: DA_PATH }); + expect(result.resp.ok).to.equal(true); + expect(result.resolvedSite).to.equal('source-site'); + expect(result.inherited).to.equal(true); + }); + + it('walks multiple source sites until one holds the file', async () => { + stubFetch({ + org: 'multi', + config: [ + { source: 'source-site', linked: 'mid-site' }, + { source: 'mid-site', linked: 'linked-site' }, + ], + sources: { 'source-site': 200 }, + }); + const result = await fetchWithMsmFallback({ org: 'multi', site: 'linked-site', daPath: DA_PATH }); + expect(result.resolvedSite).to.equal('source-site'); + expect(result.inherited).to.equal(true); + }); + + it('returns the linked-site 404 when there is no MSM config', async () => { + stubFetch({ org: 'no-config', config: 404 }); + const result = await fetchWithMsmFallback({ org: 'no-config', site: 'linked-site', daPath: DA_PATH }); + expect(result.resp.status).to.equal(404); + expect(result.resolvedSite).to.equal('linked-site'); + expect(result.inherited).to.equal(false); + }); + + it('returns the linked-site 404 when no source site holds the file', async () => { + stubFetch({ org: 'none-have', config: [{ source: 'source-site', linked: 'linked-site' }] }); + const result = await fetchWithMsmFallback({ org: 'none-have', site: 'linked-site', daPath: DA_PATH }); + expect(result.resp.status).to.equal(404); + expect(result.resolvedSite).to.equal('linked-site'); + expect(result.inherited).to.equal(false); + }); + + it('does not fall back on a non-404 error and surfaces the real status', async () => { + stubFetch({ + org: 'forbidden', + config: [{ source: 'source-site', linked: 'linked-site' }], + sources: { 'linked-site': 403, 'source-site': 200 }, + }); + const result = await fetchWithMsmFallback({ org: 'forbidden', site: 'linked-site', daPath: DA_PATH }); + expect(result.resp.status).to.equal(403); + expect(result.resolvedSite).to.equal('linked-site'); + expect(result.inherited).to.equal(false); + // A permission/server error must not trigger a source-site read. + expect(configCalls().length).to.equal(0); + }); + + it('requests the linked site then the source site at the expected DA path', async () => { + stubFetch({ + org: 'url-check', + config: [{ source: 'source-site', linked: 'linked-site' }], + sources: { 'source-site': 200 }, + }); + await fetchWithMsmFallback({ org: 'url-check', site: 'linked-site', daPath: DA_PATH }); + const requested = globalThis.fetch.args.map(([url]) => url); + expect(requested).to.include(`${DA_ADMIN}/source/url-check/linked-site${DA_PATH}`); + expect(requested).to.include(`${DA_ADMIN}/source/url-check/source-site${DA_PATH}`); + }); + + it('forwards fetch opts (headers) to the source request', async () => { + stubFetch({ org: 'opts', sources: { 'linked-site': 200 } }); + const opts = { headers: { 'Cache-Control': 'no-cache' } }; + await fetchWithMsmFallback({ org: 'opts', site: 'linked-site', daPath: DA_PATH, opts }); + const sourceCall = globalThis.fetch.args.find(([url]) => url.includes('/source/opts/linked-site/')); + expect(sourceCall[1].headers['Cache-Control']).to.equal('no-cache'); + }); + }); +});