perf(adapters): fetch Strapi, WordPress, Contentful, Sanity concurrently (#12)

CHANGELOG.md

@@ -8,9 +8,10 @@ history stays consistent across GitHub and npm.
 
 ### Changed
 
-- The Directus and Payload adapters now fetch their collections concurrently
-  with a bounded pool (`mapWithConcurrency`, exported from `@cms-lab/core`),
-  speeding up scans of multi-collection catalogs. Document order is preserved.
+- All CMS adapters (Prismic aside, which has a single endpoint) now fetch their
+  collections, content types, and single types concurrently with a bounded pool
+  (`mapWithConcurrency`, exported from `@cms-lab/core`), speeding up scans of
+  multi-collection catalogs. Document order is preserved.
 
 ### Added
 

packages/contentful/src/index.ts

@@ -1,12 +1,15 @@
 import {
   CmsFetchError,
+  mapWithConcurrency,
   readCmsDataPath,
   type CMSDocument,
   type ContentfulCmsProviderConfig,
   type ContentfulContentTypeConfig,
   type FetchLike,
 } from "@cms-lab/core";
 
+const COLLECTION_CONCURRENCY = 6;
+
 type ContentfulResponse = {
   items?: unknown[];
   skip?: number;
@@ -37,40 +40,46 @@ export async function fetchContentfulDocuments(
   options: FetchContentfulDocumentsOptions = {},
 ): Promise<CMSDocument[]> {
   const fetchImpl = options.fetch ?? fetch;
-  const documents: CMSDocument[] = [];
-
-  for (const contentType of config.contentTypes) {
-    let skip = 0;
-
-    while (true) {
-      const url = new URL(
-        `/spaces/${encodeURIComponent(config.spaceId)}/environments/${encodeURIComponent(config.environment ?? defaultEnvironment)}/entries`,
-        config.apiUrl ?? defaultApiUrl,
-      );
-      url.searchParams.set("content_type", contentType.contentType);
-      url.searchParams.set("limit", String(pageSize));
-      url.searchParams.set("skip", String(skip));
-
-      const response = await fetchJson<ContentfulResponse>(
-        fetchImpl,
-        url,
-        authHeaders(config.accessToken),
-      );
-      const rows = response.items ?? [];
-      documents.push(
-        ...rows.map((entry) => normalizeContentfulEntry(contentType, entry)),
-      );
-
-      skip += rows.length;
-      const total = response.total ?? skip;
-
-      if (rows.length === 0 || skip >= total) {
-        break;
+
+  const perContentType = await mapWithConcurrency(
+    config.contentTypes,
+    COLLECTION_CONCURRENCY,
+    async (contentType) => {
+      const documents: CMSDocument[] = [];
+      let skip = 0;
+
+      while (true) {
+        const url = new URL(
+          `/spaces/${encodeURIComponent(config.spaceId)}/environments/${encodeURIComponent(config.environment ?? defaultEnvironment)}/entries`,
+          config.apiUrl ?? defaultApiUrl,
+        );
+        url.searchParams.set("content_type", contentType.contentType);
+        url.searchParams.set("limit", String(pageSize));
+        url.searchParams.set("skip", String(skip));
+
+        const response = await fetchJson<ContentfulResponse>(
+          fetchImpl,
+          url,
+          authHeaders(config.accessToken),
+        );
+        const rows = response.items ?? [];
+        documents.push(
+          ...rows.map((entry) => normalizeContentfulEntry(contentType, entry)),
+        );
+
+        skip += rows.length;
+        const total = response.total ?? skip;
+
+        if (rows.length === 0 || skip >= total) {
+          break;
+        }
       }
-    }
-  }
 
-  return documents;
+      return documents;
+    },
+  );
+
+  return perContentType.flat();
 }
 
 export function normalizeContentfulEntry(

packages/sanity/src/index.ts

@@ -1,12 +1,15 @@
 import {
   CmsFetchError,
+  mapWithConcurrency,
   readCmsDataPath,
   type CMSDocument,
   type FetchLike,
   type SanityCmsProviderConfig,
   type SanityContentTypeConfig,
 } from "@cms-lab/core";
 
+const COLLECTION_CONCURRENCY = 6;
+
 type SanityResponse = {
   result?: unknown;
 };
@@ -22,24 +25,29 @@ export async function fetchSanityDocuments(
   options: FetchSanityDocumentsOptions = {},
 ): Promise<CMSDocument[]> {
   const fetchImpl = options.fetch ?? fetch;
-  const documents: CMSDocument[] = [];
-
-  for (const contentType of config.contentTypes) {
-    const url = sanityQueryUrl(config);
-    url.searchParams.set("query", "*[_type == $type]");
-    url.searchParams.set("$type", JSON.stringify(contentType.documentType));
-    url.searchParams.set("perspective", config.perspective ?? "published");
-
-    const response = await fetchJson<SanityResponse>(
-      fetchImpl,
-      url,
-      authHeaders(config.token),
-    );
-    const rows = Array.isArray(response.result) ? response.result : [];
-    documents.push(
-      ...rows.map((document) => normalizeSanityDocument(contentType, document)),
-    );
-  }
+
+  const perContentType = await mapWithConcurrency(
+    config.contentTypes,
+    COLLECTION_CONCURRENCY,
+    async (contentType) => {
+      const url = sanityQueryUrl(config);
+      url.searchParams.set("query", "*[_type == $type]");
+      url.searchParams.set("$type", JSON.stringify(contentType.documentType));
+      url.searchParams.set("perspective", config.perspective ?? "published");
+
+      const response = await fetchJson<SanityResponse>(
+        fetchImpl,
+        url,
+        authHeaders(config.token),
+      );
+      const rows = Array.isArray(response.result) ? response.result : [];
+      return rows.map((document) =>
+        normalizeSanityDocument(contentType, document),
+      );
+    },
+  );
+
+  const documents = perContentType.flat();
 
   await hydrateImageAssetMetadata(config, fetchImpl, documents);
 

packages/strapi/src/index.ts

@@ -1,5 +1,6 @@
 import {
   CmsFetchError,
+  mapWithConcurrency,
   readCmsDataPath,
   type CMSDocument,
   type FetchLike,
@@ -8,6 +9,8 @@ import {
   type StrapiSingleTypeConfig,
 } from "@cms-lab/core";
 
+const COLLECTION_CONCURRENCY = 6;
+
 type StrapiResponse = {
   data?: unknown[];
   meta?: {
@@ -31,61 +34,73 @@ export async function fetchStrapiDocuments(
   options: FetchStrapiDocumentsOptions = {},
 ): Promise<CMSDocument[]> {
   const fetchImpl = options.fetch ?? fetch;
-  const documents: CMSDocument[] = [];
 
-  for (const collection of config.collections ?? []) {
-    let page = 1;
+  const collectionDocuments = await mapWithConcurrency(
+    config.collections ?? [],
+    COLLECTION_CONCURRENCY,
+    async (collection) => {
+      const documents: CMSDocument[] = [];
+      let page = 1;
+
+      while (true) {
+        const url = strapiEndpointUrl(config.url, collection.endpoint);
+        url.searchParams.set("pagination[pageSize]", "100");
+        url.searchParams.set("pagination[page]", String(page));
+        url.searchParams.set("populate", "*");
+        applyLocale(url, collection.locale ?? config.locale);
+
+        const response = await fetchJson<StrapiResponse>(
+          fetchImpl,
+          url,
+          authHeaders(config.token),
+        );
+        documents.push(
+          ...(response.data ?? []).map((item) =>
+            normalizeStrapiItem(collection, item, { entryKind: "collection" }),
+          ),
+        );
+        const pageCount = response.meta?.pagination?.pageCount ?? page;
+
+        if (page >= pageCount) {
+          break;
+        }
+
+        page += 1;
+      }
+
+      return documents;
+    },
+  );
 
-    while (true) {
-      const url = strapiEndpointUrl(config.url, collection.endpoint);
-      url.searchParams.set("pagination[pageSize]", "100");
-      url.searchParams.set("pagination[page]", String(page));
+  const singleTypeDocuments = await mapWithConcurrency(
+    config.singleTypes ?? [],
+    COLLECTION_CONCURRENCY,
+    async (singleType) => {
+      const url = strapiEndpointUrl(config.url, singleType.endpoint);
       url.searchParams.set("populate", "*");
-      applyLocale(url, collection.locale ?? config.locale);
+      applyLocale(url, singleType.locale ?? config.locale);
 
-      const response = await fetchJson<StrapiResponse>(
+      const response = await fetchJson<StrapiSingleTypeResponse>(
         fetchImpl,
         url,
         authHeaders(config.token),
       );
-      documents.push(
-        ...(response.data ?? []).map((item) =>
-          normalizeStrapiItem(collection, item, { entryKind: "collection" }),
-        ),
-      );
-      const pageCount = response.meta?.pagination?.pageCount ?? page;
 
-      if (page >= pageCount) {
-        break;
+      if (response.data != null) {
+        return [
+          normalizeStrapiItem(singleType, response.data, {
+            fallbackUid: false,
+            routable: false,
+            entryKind: "single",
+          }),
+        ];
       }
 
-      page += 1;
-    }
-  }
-
-  for (const singleType of config.singleTypes ?? []) {
-    const url = strapiEndpointUrl(config.url, singleType.endpoint);
-    url.searchParams.set("populate", "*");
-    applyLocale(url, singleType.locale ?? config.locale);
-
-    const response = await fetchJson<StrapiSingleTypeResponse>(
-      fetchImpl,
-      url,
-      authHeaders(config.token),
-    );
-
-    if (response.data != null) {
-      documents.push(
-        normalizeStrapiItem(singleType, response.data, {
-          fallbackUid: false,
-          routable: false,
-          entryKind: "single",
-        }),
-      );
-    }
-  }
+      return [];
+    },
+  );
 
-  return documents;
+  return [...collectionDocuments.flat(), ...singleTypeDocuments.flat()];
 }
 
 export function normalizeStrapiItem(

packages/wordpress/src/index.ts

@@ -1,12 +1,15 @@
 import {
   CmsFetchError,
+  mapWithConcurrency,
   readCmsDataPath,
   type CMSDocument,
   type FetchLike,
   type WordPressCmsProviderConfig,
   type WordPressContentTypeConfig,
 } from "@cms-lab/core";
 
+const COLLECTION_CONCURRENCY = 6;
+
 type WordPressItem = Record<string, unknown>;
 
 export type FetchWordPressDocumentsOptions = {
@@ -23,34 +26,40 @@ export async function fetchWordPressDocuments(
   options: FetchWordPressDocumentsOptions = {},
 ): Promise<CMSDocument[]> {
   const fetchImpl = options.fetch ?? fetch;
-  const documents: CMSDocument[] = [];
-
-  for (const contentType of config.contentTypes ?? defaultContentTypes) {
-    let page = 1;
-
-    while (true) {
-      const url = new URL(
-        `/wp-json/wp/v2/${trimSlashes(contentType.endpoint)}`,
-        config.url,
-      );
-      url.searchParams.set("per_page", "100");
-      url.searchParams.set("page", String(page));
-
-      const { rows, pages } = await fetchRows(fetchImpl, url);
-      documents.push(
-        ...rows.map((row) => normalizeWordPressItem(contentType, row)),
-      );
-      const totalPages = pages;
-
-      if (page >= totalPages) {
-        break;
+
+  const perContentType = await mapWithConcurrency(
+    config.contentTypes ?? defaultContentTypes,
+    COLLECTION_CONCURRENCY,
+    async (contentType) => {
+      const documents: CMSDocument[] = [];
+      let page = 1;
+
+      while (true) {
+        const url = new URL(
+          `/wp-json/wp/v2/${trimSlashes(contentType.endpoint)}`,
+          config.url,
+        );
+        url.searchParams.set("per_page", "100");
+        url.searchParams.set("page", String(page));
+
+        const { rows, pages } = await fetchRows(fetchImpl, url);
+        documents.push(
+          ...rows.map((row) => normalizeWordPressItem(contentType, row)),
+        );
+        const totalPages = pages;
+
+        if (page >= totalPages) {
+          break;
+        }
+
+        page += 1;
       }
 
-      page += 1;
-    }
-  }
+      return documents;
+    },
+  );
 
-  return documents;
+  return perContentType.flat();
 }
 
 async function fetchRows(