diff --git a/CHANGELOG.md b/CHANGELOG.md index 905a3e6..1216ed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ history stays consistent across GitHub and npm. ### Changed -- The Directus and Payload adapters now fetch their collections concurrently - with a bounded pool (`mapWithConcurrency`, exported from `@cms-lab/core`), - speeding up scans of multi-collection catalogs. Document order is preserved. +- All CMS adapters (Prismic aside, which has a single endpoint) now fetch their + collections, content types, and single types concurrently with a bounded pool + (`mapWithConcurrency`, exported from `@cms-lab/core`), speeding up scans of + multi-collection catalogs. Document order is preserved. ### Added diff --git a/packages/contentful/src/index.ts b/packages/contentful/src/index.ts index 5a9ce31..f076754 100644 --- a/packages/contentful/src/index.ts +++ b/packages/contentful/src/index.ts @@ -1,5 +1,6 @@ import { CmsFetchError, + mapWithConcurrency, readCmsDataPath, type CMSDocument, type ContentfulCmsProviderConfig, @@ -7,6 +8,8 @@ import { type FetchLike, } from "@cms-lab/core"; +const COLLECTION_CONCURRENCY = 6; + type ContentfulResponse = { items?: unknown[]; skip?: number; @@ -37,40 +40,46 @@ export async function fetchContentfulDocuments( options: FetchContentfulDocumentsOptions = {}, ): Promise { const fetchImpl = options.fetch ?? fetch; - const documents: CMSDocument[] = []; - - for (const contentType of config.contentTypes) { - let skip = 0; - - while (true) { - const url = new URL( - `/spaces/${encodeURIComponent(config.spaceId)}/environments/${encodeURIComponent(config.environment ?? defaultEnvironment)}/entries`, - config.apiUrl ?? defaultApiUrl, - ); - url.searchParams.set("content_type", contentType.contentType); - url.searchParams.set("limit", String(pageSize)); - url.searchParams.set("skip", String(skip)); - - const response = await fetchJson( - fetchImpl, - url, - authHeaders(config.accessToken), - ); - const rows = response.items ?? []; - documents.push( - ...rows.map((entry) => normalizeContentfulEntry(contentType, entry)), - ); - - skip += rows.length; - const total = response.total ?? skip; - - if (rows.length === 0 || skip >= total) { - break; + + const perContentType = await mapWithConcurrency( + config.contentTypes, + COLLECTION_CONCURRENCY, + async (contentType) => { + const documents: CMSDocument[] = []; + let skip = 0; + + while (true) { + const url = new URL( + `/spaces/${encodeURIComponent(config.spaceId)}/environments/${encodeURIComponent(config.environment ?? defaultEnvironment)}/entries`, + config.apiUrl ?? defaultApiUrl, + ); + url.searchParams.set("content_type", contentType.contentType); + url.searchParams.set("limit", String(pageSize)); + url.searchParams.set("skip", String(skip)); + + const response = await fetchJson( + fetchImpl, + url, + authHeaders(config.accessToken), + ); + const rows = response.items ?? []; + documents.push( + ...rows.map((entry) => normalizeContentfulEntry(contentType, entry)), + ); + + skip += rows.length; + const total = response.total ?? skip; + + if (rows.length === 0 || skip >= total) { + break; + } } - } - } - return documents; + return documents; + }, + ); + + return perContentType.flat(); } export function normalizeContentfulEntry( diff --git a/packages/sanity/src/index.ts b/packages/sanity/src/index.ts index 86e1313..714d1af 100644 --- a/packages/sanity/src/index.ts +++ b/packages/sanity/src/index.ts @@ -1,5 +1,6 @@ import { CmsFetchError, + mapWithConcurrency, readCmsDataPath, type CMSDocument, type FetchLike, @@ -7,6 +8,8 @@ import { type SanityContentTypeConfig, } from "@cms-lab/core"; +const COLLECTION_CONCURRENCY = 6; + type SanityResponse = { result?: unknown; }; @@ -22,24 +25,29 @@ export async function fetchSanityDocuments( options: FetchSanityDocumentsOptions = {}, ): Promise { const fetchImpl = options.fetch ?? fetch; - const documents: CMSDocument[] = []; - - for (const contentType of config.contentTypes) { - const url = sanityQueryUrl(config); - url.searchParams.set("query", "*[_type == $type]"); - url.searchParams.set("$type", JSON.stringify(contentType.documentType)); - url.searchParams.set("perspective", config.perspective ?? "published"); - - const response = await fetchJson( - fetchImpl, - url, - authHeaders(config.token), - ); - const rows = Array.isArray(response.result) ? response.result : []; - documents.push( - ...rows.map((document) => normalizeSanityDocument(contentType, document)), - ); - } + + const perContentType = await mapWithConcurrency( + config.contentTypes, + COLLECTION_CONCURRENCY, + async (contentType) => { + const url = sanityQueryUrl(config); + url.searchParams.set("query", "*[_type == $type]"); + url.searchParams.set("$type", JSON.stringify(contentType.documentType)); + url.searchParams.set("perspective", config.perspective ?? "published"); + + const response = await fetchJson( + fetchImpl, + url, + authHeaders(config.token), + ); + const rows = Array.isArray(response.result) ? response.result : []; + return rows.map((document) => + normalizeSanityDocument(contentType, document), + ); + }, + ); + + const documents = perContentType.flat(); await hydrateImageAssetMetadata(config, fetchImpl, documents); diff --git a/packages/strapi/src/index.ts b/packages/strapi/src/index.ts index f6fc0b9..1fae7c5 100644 --- a/packages/strapi/src/index.ts +++ b/packages/strapi/src/index.ts @@ -1,5 +1,6 @@ import { CmsFetchError, + mapWithConcurrency, readCmsDataPath, type CMSDocument, type FetchLike, @@ -8,6 +9,8 @@ import { type StrapiSingleTypeConfig, } from "@cms-lab/core"; +const COLLECTION_CONCURRENCY = 6; + type StrapiResponse = { data?: unknown[]; meta?: { @@ -31,61 +34,73 @@ export async function fetchStrapiDocuments( options: FetchStrapiDocumentsOptions = {}, ): Promise { const fetchImpl = options.fetch ?? fetch; - const documents: CMSDocument[] = []; - for (const collection of config.collections ?? []) { - let page = 1; + const collectionDocuments = await mapWithConcurrency( + config.collections ?? [], + COLLECTION_CONCURRENCY, + async (collection) => { + const documents: CMSDocument[] = []; + let page = 1; + + while (true) { + const url = strapiEndpointUrl(config.url, collection.endpoint); + url.searchParams.set("pagination[pageSize]", "100"); + url.searchParams.set("pagination[page]", String(page)); + url.searchParams.set("populate", "*"); + applyLocale(url, collection.locale ?? config.locale); + + const response = await fetchJson( + fetchImpl, + url, + authHeaders(config.token), + ); + documents.push( + ...(response.data ?? []).map((item) => + normalizeStrapiItem(collection, item, { entryKind: "collection" }), + ), + ); + const pageCount = response.meta?.pagination?.pageCount ?? page; + + if (page >= pageCount) { + break; + } + + page += 1; + } + + return documents; + }, + ); - while (true) { - const url = strapiEndpointUrl(config.url, collection.endpoint); - url.searchParams.set("pagination[pageSize]", "100"); - url.searchParams.set("pagination[page]", String(page)); + const singleTypeDocuments = await mapWithConcurrency( + config.singleTypes ?? [], + COLLECTION_CONCURRENCY, + async (singleType) => { + const url = strapiEndpointUrl(config.url, singleType.endpoint); url.searchParams.set("populate", "*"); - applyLocale(url, collection.locale ?? config.locale); + applyLocale(url, singleType.locale ?? config.locale); - const response = await fetchJson( + const response = await fetchJson( fetchImpl, url, authHeaders(config.token), ); - documents.push( - ...(response.data ?? []).map((item) => - normalizeStrapiItem(collection, item, { entryKind: "collection" }), - ), - ); - const pageCount = response.meta?.pagination?.pageCount ?? page; - if (page >= pageCount) { - break; + if (response.data != null) { + return [ + normalizeStrapiItem(singleType, response.data, { + fallbackUid: false, + routable: false, + entryKind: "single", + }), + ]; } - page += 1; - } - } - - for (const singleType of config.singleTypes ?? []) { - const url = strapiEndpointUrl(config.url, singleType.endpoint); - url.searchParams.set("populate", "*"); - applyLocale(url, singleType.locale ?? config.locale); - - const response = await fetchJson( - fetchImpl, - url, - authHeaders(config.token), - ); - - if (response.data != null) { - documents.push( - normalizeStrapiItem(singleType, response.data, { - fallbackUid: false, - routable: false, - entryKind: "single", - }), - ); - } - } + return []; + }, + ); - return documents; + return [...collectionDocuments.flat(), ...singleTypeDocuments.flat()]; } export function normalizeStrapiItem( diff --git a/packages/wordpress/src/index.ts b/packages/wordpress/src/index.ts index 175579b..b142b1d 100644 --- a/packages/wordpress/src/index.ts +++ b/packages/wordpress/src/index.ts @@ -1,5 +1,6 @@ import { CmsFetchError, + mapWithConcurrency, readCmsDataPath, type CMSDocument, type FetchLike, @@ -7,6 +8,8 @@ import { type WordPressContentTypeConfig, } from "@cms-lab/core"; +const COLLECTION_CONCURRENCY = 6; + type WordPressItem = Record; export type FetchWordPressDocumentsOptions = { @@ -23,34 +26,40 @@ export async function fetchWordPressDocuments( options: FetchWordPressDocumentsOptions = {}, ): Promise { const fetchImpl = options.fetch ?? fetch; - const documents: CMSDocument[] = []; - - for (const contentType of config.contentTypes ?? defaultContentTypes) { - let page = 1; - - while (true) { - const url = new URL( - `/wp-json/wp/v2/${trimSlashes(contentType.endpoint)}`, - config.url, - ); - url.searchParams.set("per_page", "100"); - url.searchParams.set("page", String(page)); - - const { rows, pages } = await fetchRows(fetchImpl, url); - documents.push( - ...rows.map((row) => normalizeWordPressItem(contentType, row)), - ); - const totalPages = pages; - - if (page >= totalPages) { - break; + + const perContentType = await mapWithConcurrency( + config.contentTypes ?? defaultContentTypes, + COLLECTION_CONCURRENCY, + async (contentType) => { + const documents: CMSDocument[] = []; + let page = 1; + + while (true) { + const url = new URL( + `/wp-json/wp/v2/${trimSlashes(contentType.endpoint)}`, + config.url, + ); + url.searchParams.set("per_page", "100"); + url.searchParams.set("page", String(page)); + + const { rows, pages } = await fetchRows(fetchImpl, url); + documents.push( + ...rows.map((row) => normalizeWordPressItem(contentType, row)), + ); + const totalPages = pages; + + if (page >= totalPages) { + break; + } + + page += 1; } - page += 1; - } - } + return documents; + }, + ); - return documents; + return perContentType.flat(); } async function fetchRows(