From 5fd71f9c50adbe74bf8e743a8d2f4502f791aad5 Mon Sep 17 00:00:00 2001 From: Clinton Lunn Date: Fri, 14 Aug 2026 13:16:02 -0600 Subject: [PATCH 1/6] fix(stac): page static catalog searches instead of reporting page size as total --- .../src/components/layout/TopToolbar.tsx | 1 + .../geolibre-desktop/src/i18n/locales/en.json | 1 + packages/plugins/src/plugins/maplibre-stac.ts | 25 ++- packages/plugins/src/plugins/stac-api.ts | 144 ++++++++++++------ tests/stac-api.test.ts | 51 +++++++ 5 files changed, 167 insertions(+), 55 deletions(-) diff --git a/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx b/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx index 33106940b9..0b7660c556 100644 --- a/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx +++ b/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx @@ -906,6 +906,7 @@ export function TopToolbar({ resultsCleared: t("stacPlugin.resultsCleared"), searching: t("stacPlugin.searching"), loadingMore: t("stacPlugin.loadingMore"), + noNewItems: t("stacPlugin.noNewItems"), noResults: t("stacPlugin.noResults"), searchFailed: t("stacPlugin.searchFailed"), showing: (count) => t("stacPlugin.showing", { count }), diff --git a/apps/geolibre-desktop/src/i18n/locales/en.json b/apps/geolibre-desktop/src/i18n/locales/en.json index 2bc06f1ad4..8c5975dddc 100644 --- a/apps/geolibre-desktop/src/i18n/locales/en.json +++ b/apps/geolibre-desktop/src/i18n/locales/en.json @@ -3399,6 +3399,7 @@ "resultsCleared": "Search results cleared.", "searching": "Searching STAC items…", "loadingMore": "Loading more items…", + "noNewItems": "No new items in that part of the catalog. Load more to keep searching.", "noResults": "No STAC items matched these filters.", "searchFailed": "STAC search failed", "showing": "Showing {{count}} items.", diff --git a/packages/plugins/src/plugins/maplibre-stac.ts b/packages/plugins/src/plugins/maplibre-stac.ts index 829e3c4d2b..7157174821 100644 --- a/packages/plugins/src/plugins/maplibre-stac.ts +++ b/packages/plugins/src/plugins/maplibre-stac.ts @@ -15,6 +15,8 @@ import { type StacIndexCatalog, type StacItem, type StacNextPage, + type StacSearchResult, + type StacWalkCursor, } from "./stac-api"; export const STAC_PLUGIN_ID = "geolibre-stac-catalogs"; @@ -111,6 +113,7 @@ export interface StacLabels { resultsCleared: string; searching: string; loadingMore: string; + noNewItems: string; noResults: string; searchFailed: string; loadMore: string; @@ -178,6 +181,7 @@ let labels: StacLabels = { resultsCleared: "Search results cleared.", searching: "Searching STAC items…", loadingMore: "Loading more items…", + noNewItems: "No new items in that part of the catalog. Load more to keep searching.", noResults: "No STAC items matched these filters.", searchFailed: "STAC search failed", loadMore: "Load more", @@ -661,6 +665,7 @@ function buildPanel(container: HTMLElement): () => void { let filtered: StacIndexCatalog[] = []; let connection: StacConnection | null = null; let nextPage: StacNextPage | undefined; + let walkCursor: StacWalkCursor | undefined; let allItems: StacItem[] = []; let searchGeneration = 0; let cancelDraw: (() => void) | null = null; @@ -714,6 +719,7 @@ function buildPanel(container: HTMLElement): () => void { searchGeneration += 1; allItems = []; nextPage = undefined; + walkCursor = undefined; results.innerHTML = ""; cardsByItemId.clear(); selectItem(null, false); @@ -854,6 +860,13 @@ function buildPanel(container: HTMLElement): () => void { return parsed as Record; }; + const searchStatus = (append: boolean, result: StacSearchResult): string => { + if (append && !result.items.length && walkCursor) return labels.noNewItems; + if (!allItems.length) return labels.noResults; + if (result.matched) return labels.showingOfMatched(allItems.length, result.matched); + return labels.showing(allItems.length); + }; + const runSearch = async (append: boolean): Promise => { if (!connection) return; const generation = ++searchGeneration; @@ -874,6 +887,7 @@ function buildPanel(container: HTMLElement): () => void { additional: parseAdditionalParams(), limit: 20, next: append ? nextPage : undefined, + cursor: append ? walkCursor : undefined, signal: controller.signal, }; const response = connection.isApi @@ -882,20 +896,15 @@ function buildPanel(container: HTMLElement): () => void { if (generation !== searchGeneration) return; allItems = append ? [...allItems, ...response.items] : response.items; nextPage = response.next; + walkCursor = response.cursor; // A fresh search invalidates the selection; "Load more" keeps it. if (!append) selectedItemId = null; renderItems(); showFootprints(allItems); applySelection(false); - loadMore.hidden = !nextPage; + loadMore.hidden = !nextPage && !walkCursor; clearResultsButton.disabled = allItems.length === 0; - setStatus( - allItems.length - ? response.matched - ? labels.showingOfMatched(allItems.length, response.matched) - : labels.showing(allItems.length) - : labels.noResults, - ); + setStatus(searchStatus(append, response)); } catch (error) { setStatus(error instanceof Error ? error.message : labels.searchFailed, true); } finally { diff --git a/packages/plugins/src/plugins/stac-api.ts b/packages/plugins/src/plugins/stac-api.ts index 3510495ebe..090e2ef771 100644 --- a/packages/plugins/src/plugins/stac-api.ts +++ b/packages/plugins/src/plugins/stac-api.ts @@ -1,6 +1,9 @@ import type { BBox, Feature, Geometry } from "geojson"; export const STAC_INDEX_CATALOGS_URL = "https://stacindex.org/api/catalogs"; +// No item-search endpoint to ask, so a page is however much of the tree the walk covers. +const STATIC_SEARCH_READS_PER_PAGE = 300; +const STATIC_SEARCH_CONCURRENCY = 12; export interface StacIndexCatalog { id: number; @@ -56,10 +59,20 @@ export interface StacConnection { root: Record; } +/** A walk in progress; hand it back to continue. Mutated in place rather than copied. */ +export interface StacWalkCursor { + items: Unread[]; + folders: Unread[]; + visited: Set; + /** Items already delivered, so the last page can report a real total. */ + offset: number; +} + export interface StacSearchOptions { bbox?: [number, number, number, number]; datetime?: string; collections?: string[]; + cursor?: StacWalkCursor; /** Additional STAC API Item Search members such as query, filter, sortby, or fields. */ additional?: Record; limit?: number; @@ -76,6 +89,7 @@ export interface StacNextPage { export interface StacSearchResult { items: StacItem[]; next?: StacNextPage; + cursor?: StacWalkCursor; matched?: number; } @@ -109,7 +123,7 @@ export function browserAssetHref(href: string, base: string): string { return `https://${bucket}.s3.amazonaws.com${url.pathname}${url.search}${url.hash}`; } -async function fetchJson(url: string, init: RequestInit, fetcher: FetchLike): Promise { +async function fetchJson(url: string, init: RequestInit, fetcher: FetchLike): Promise { const response = await fetcher(url, { ...init, headers: { Accept: "application/geo+json, application/json", ...init.headers }, @@ -148,6 +162,13 @@ function linksOf(value: unknown, base: string): StacLink[] { }); } +function isStacItem(value: unknown): value is StacItem { + if (typeof value !== "object" || value === null) return false; + return ( + "id" in value && typeof value.id === "string" && "assets" in value && Boolean(value.assets) + ); +} + function normalizeItem(item: StacItem, base: string): StacItem { const assets = Object.fromEntries( Object.entries(item.assets ?? {}).flatMap(([key, asset]) => { @@ -169,9 +190,9 @@ export async function connectStac( ): Promise { if (!httpUrl(inputUrl)) throw new Error("Enter a valid HTTP or HTTPS STAC URL"); const url = new URL(inputUrl).href; - const raw = await fetchJson(url, { signal }, fetcher); - if (!raw || typeof raw !== "object") throw new Error("The URL did not return a STAC document"); - const root = raw as Record; + const root = await fetchJson>(url, { signal }, fetcher); + if (typeof root !== "object" || root === null) + throw new Error("The URL did not return a STAC document"); const links = linksOf(root.links, url); const conforms = Array.isArray(root.conformsTo) ? root.conformsTo.map(String) : []; const searchLink = links.find((link) => link.rel === "search"); @@ -186,9 +207,11 @@ export async function connectStac( ); if (collectionsLink) { try { - const data = (await fetchJson(collectionsLink.href, { signal }, fetcher)) as { - collections?: StacCollection[]; - }; + const data = await fetchJson<{ collections?: StacCollection[] }>( + collectionsLink.href, + { signal }, + fetcher, + ); if (Array.isArray(data.collections)) collections = data.collections; } catch { // Collection discovery is helpful but not required for item search. @@ -213,15 +236,7 @@ function parseItems(raw: unknown, responseUrl: string): StacSearchResult { throw new Error("The STAC server returned invalid search data"); const data = raw as Record; const features = Array.isArray(data.features) ? data.features : []; - const items = features - .filter( - (feature): feature is StacItem => - Boolean(feature) && - typeof feature === "object" && - typeof (feature as StacItem).id === "string" && - Boolean((feature as StacItem).assets), - ) - .map((item) => normalizeItem(item, responseUrl)); + const items = features.filter(isStacItem).map((item) => normalizeItem(item, responseUrl)); const nextLink = linksOf(data.links, responseUrl).find((link) => link.rel === "next"); const context = data.context as { matched?: unknown } | undefined; const numberMatched = data.numberMatched; @@ -313,46 +328,81 @@ function inTime(item: StacItem, interval?: string): boolean { } /** Searches a static catalog by following child/item links, with a hard safety cap. */ +/** Queued but unread; the root arrives already read. */ +type Unread = { url: string; document?: Record }; + export async function searchStaticStac( connection: StacConnection, options: StacSearchOptions, fetcher: FetchLike = fetch, ): Promise { - const queue: Array<{ url: string; document?: Record }> = [ - { url: connection.url, document: connection.root }, - ]; - const visited = new Set(); - const items: StacItem[] = []; + const walk = options.cursor ?? { + items: [], + folders: [{ url: connection.url, document: connection.root }], + visited: new Set(), + offset: 0, + }; + const found: StacItem[] = []; const limit = Math.max(1, Math.min(options.limit ?? 20, 100)); - while (queue.length && visited.size < 300 && items.length < limit) { - const current = queue.shift()!; - if (visited.has(current.url)) continue; - visited.add(current.url); - const document = - current.document ?? - ((await fetchJson(current.url, { signal: options.signal }, fetcher)) as Record< - string, - unknown - >); - if (document.type === "Feature") { - const item = normalizeItem(document as unknown as StacItem, current.url); - // itemBbox flattens 3D (6-element) bboxes; item.bbox[2]/[3] would be minZ/maxX there. - const bbox = itemBbox(item); - if ( - (!options.collections?.length || - (item.collection && options.collections.includes(item.collection))) && - (!options.bbox || (bbox && intersects(bbox, options.bbox))) && - inTime(item, options.datetime) - ) { - items.push(item); - } - continue; + let reads = 0; + + const accepts = (item: StacItem): boolean => { + // itemBbox flattens 3D (6-element) bboxes; item.bbox[2]/[3] would be minZ/maxX there. + const bbox = itemBbox(item); + if (options.collections?.length && !options.collections.includes(item.collection ?? "")) { + return false; } - for (const link of linksOf(document.links, current.url)) { - if (link.rel === "item" || link.rel === "child") queue.push({ url: link.href }); + if (options.bbox && !(bbox && intersects(bbox, options.bbox))) return false; + return inTime(item, options.datetime); + }; + + const takeBatch = (): Unread[] => { + const room = Math.min( + STATIC_SEARCH_CONCURRENCY, + limit - found.length, + STATIC_SEARCH_READS_PER_PAGE - reads, + ); + const batch: Unread[] = []; + while (batch.length < room && (walk.items.length || walk.folders.length)) { + const entry = (walk.items.length ? walk.items : walk.folders).shift()!; + if (walk.visited.has(entry.url)) continue; + walk.visited.add(entry.url); + batch.push(entry); } + return batch; + }; + + const read = async (entry: Unread): Promise> => + entry.document ?? + fetchJson>(entry.url, { signal: options.signal }, fetcher); + + const collect = (document: Record, url: string): void => { + if (document.type !== "Feature") { + for (const link of linksOf(document.links, url)) { + if (link.rel === "item") walk.items.push({ url: link.href }); + else if (link.rel === "child") walk.folders.push({ url: link.href }); + } + return; + } + if (!isStacItem(document)) return; + const item = normalizeItem(document, url); + if (accepts(item)) found.push(item); + }; + + while (found.length < limit && reads < STATIC_SEARCH_READS_PER_PAGE) { + const batch = takeBatch(); + if (!batch.length) break; + reads += batch.length; + const documents = await Promise.all(batch.map(read)); + documents.forEach((document, index) => collect(document, batch[index].url)); } - return { items, matched: items.length }; + + const offset = walk.offset + found.length; + const done = !walk.items.length && !walk.folders.length; + // Counting every page, not the last: the panel accumulates, so a page total reads "25 of 5". + if (done) return { items: found, matched: offset }; + walk.offset = offset; + return { items: found, cursor: walk }; } export function itemBbox(item: StacItem): [number, number, number, number] | undefined { diff --git a/tests/stac-api.test.ts b/tests/stac-api.test.ts index 5d46289578..935220a04b 100644 --- a/tests/stac-api.test.ts +++ b/tests/stac-api.test.ts @@ -241,6 +241,57 @@ test("searchStaticStac traverses child and item links and applies filters", asyn ); }); +test("searchStaticStac pages through a catalog holding more items than one page fits", async () => { + const total = 25; + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: Array.from({ length: total }, (_value, index) => ({ + rel: "item", + href: `./item${index}.json`, + })), + }, + }; + for (let index = 0; index < total; index += 1) { + docs[`https://example.com/stac/item${index}.json`] = { + type: "Feature", + id: `item${index}`, + collection: "many", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }; + } + const fetcher = (async (input: RequestInfo | URL) => + jsonResponse(docs[String(input)])) as typeof fetch; + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }; + + const first = await searchStaticStac(connection, { limit: 10 }, fetcher); + assert.equal(first.items.length, 10); + assert.ok(first.cursor, "a walk with documents left over reports where it stopped"); + assert.equal(first.matched, undefined); + + const second = await searchStaticStac(connection, { limit: 10, cursor: first.cursor }, fetcher); + assert.equal(second.items.length, 10); + assert.deepEqual( + second.items.map((item) => item.id).filter((id) => first.items.some((seen) => seen.id === id)), + [], + "a resumed page repeats nothing from the page before it", + ); + + const third = await searchStaticStac(connection, { limit: 10, cursor: second.cursor }, fetcher); + assert.equal(third.items.length, 5); + assert.equal(third.cursor, undefined, "the walk is done, so there is nothing to resume"); + assert.equal(third.matched, 5); +}); + test("asset and bbox helpers recognize common STAC data", () => { assert.equal(isVisualizableAsset({ href: "https://example.com/a.TIF?download=1" }), true); assert.equal(isVisualizableAsset({ href: "https://example.com/data.bin" }), false); From d24f89a2c4d67a4c3c7df34bb14384871ed98fb4 Mon Sep 17 00:00:00 2001 From: Clinton Lunn Date: Fri, 14 Aug 2026 18:21:33 -0600 Subject: [PATCH 2/6] fix(stac): keep the filters from collapsing as results accumulate --- packages/plugins/src/plugins/maplibre-stac.ts | 3 +- tests/stac-api.test.ts | 92 ++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/plugins/src/plugins/maplibre-stac.ts b/packages/plugins/src/plugins/maplibre-stac.ts index 7157174821..908a732412 100644 --- a/packages/plugins/src/plugins/maplibre-stac.ts +++ b/packages/plugins/src/plugins/maplibre-stac.ts @@ -249,7 +249,8 @@ const style = { // the controls above it scroll as a group rather than pushing it off-panel. results: "display:flex;flex:1 1 auto;min-height:150px;overflow:auto;flex-direction:column;gap:7px;", - controls: "display:flex;flex-direction:column;gap:10px;flex:0 1 auto;min-height:0;overflow:auto;", + controls: + "display:flex;flex-direction:column;gap:10px;flex:0 1 auto;min-height:180px;overflow:auto;", card: "display:flex;flex-direction:column;gap:5px;padding:8px;border:1px solid hsl(var(--border));" + "border-radius:7px;background:hsl(var(--muted));", diff --git a/tests/stac-api.test.ts b/tests/stac-api.test.ts index 935220a04b..010417a80d 100644 --- a/tests/stac-api.test.ts +++ b/tests/stac-api.test.ts @@ -289,7 +289,97 @@ test("searchStaticStac pages through a catalog holding more items than one page const third = await searchStaticStac(connection, { limit: 10, cursor: second.cursor }, fetcher); assert.equal(third.items.length, 5); assert.equal(third.cursor, undefined, "the walk is done, so there is nothing to resume"); - assert.equal(third.matched, 5); + assert.equal(third.matched, 25); +}); + +test("searchStaticStac reads items before folders, so a page is not spent on structure", async () => { + // Items one folder down, behind a hundred empty ones. Discovery order spends the page on + // folders and returns nothing. + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: [ + { rel: "child", href: "./has-items.json" }, + ...Array.from({ length: 100 }, (_value, index) => ({ + rel: "child", + href: `./empty${index}.json`, + })), + ], + }, + "https://example.com/stac/has-items.json": { + type: "Catalog", + links: Array.from({ length: 3 }, (_value, index) => ({ + rel: "item", + href: `./item${index}.json`, + })), + }, + }; + for (let index = 0; index < 100; index += 1) { + docs[`https://example.com/stac/empty${index}.json`] = { type: "Catalog", links: [] }; + } + for (let index = 0; index < 3; index += 1) { + docs[`https://example.com/stac/item${index}.json`] = { + type: "Feature", + id: `item${index}`, + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }; + } + let reads = 0; + const fetcher = (async (input: RequestInfo | URL) => { + reads += 1; + return jsonResponse(docs[String(input)]); + }) as typeof fetch; + + const result = await searchStaticStac( + { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }, + { limit: 3 }, + fetcher, + ); + + assert.equal(result.items.length, 3); + // Discovery order costs a hundred more. + assert.ok(reads < 30, `expected the items to be reached quickly, took ${reads} reads`); +}); + +test("a page stops reading at its budget rather than crawling the whole catalog", async () => { + // No items anywhere, so only the budget can end the page. + let reads = 0; + const fetcher = (async () => { + reads += 1; + return jsonResponse({ + type: "Catalog", + links: [ + { rel: "child", href: `./${reads}-a.json` }, + { rel: "child", href: `./${reads}-b.json` }, + ], + }); + }) as typeof fetch; + + const result = await searchStaticStac( + { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: { type: "Catalog", links: [{ rel: "child", href: "./a.json" }] }, + }, + { limit: 20 }, + fetcher, + ); + + assert.deepEqual(result.items, []); + assert.ok(reads <= 300, `a page must stop at its budget, read ${reads}`); + assert.ok(result.cursor, "and report that the walk is unfinished"); }); test("asset and bbox helpers recognize common STAC data", () => { From dac8abd0ceadea96978faa7e7bd1771aaa25feb5 Mon Sep 17 00:00:00 2001 From: Clinton Lunn Date: Fri, 14 Aug 2026 19:47:28 -0600 Subject: [PATCH 3/6] refactor(stac): rename StacWalkCursor to StacSearchCursor --- packages/plugins/src/plugins/maplibre-stac.ts | 14 +++++++------- packages/plugins/src/plugins/stac-api.ts | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/plugins/src/plugins/maplibre-stac.ts b/packages/plugins/src/plugins/maplibre-stac.ts index 908a732412..6ce2f51c8e 100644 --- a/packages/plugins/src/plugins/maplibre-stac.ts +++ b/packages/plugins/src/plugins/maplibre-stac.ts @@ -16,7 +16,7 @@ import { type StacItem, type StacNextPage, type StacSearchResult, - type StacWalkCursor, + type StacSearchCursor, } from "./stac-api"; export const STAC_PLUGIN_ID = "geolibre-stac-catalogs"; @@ -666,7 +666,7 @@ function buildPanel(container: HTMLElement): () => void { let filtered: StacIndexCatalog[] = []; let connection: StacConnection | null = null; let nextPage: StacNextPage | undefined; - let walkCursor: StacWalkCursor | undefined; + let searchCursor: StacSearchCursor | undefined; let allItems: StacItem[] = []; let searchGeneration = 0; let cancelDraw: (() => void) | null = null; @@ -720,7 +720,7 @@ function buildPanel(container: HTMLElement): () => void { searchGeneration += 1; allItems = []; nextPage = undefined; - walkCursor = undefined; + searchCursor = undefined; results.innerHTML = ""; cardsByItemId.clear(); selectItem(null, false); @@ -862,7 +862,7 @@ function buildPanel(container: HTMLElement): () => void { }; const searchStatus = (append: boolean, result: StacSearchResult): string => { - if (append && !result.items.length && walkCursor) return labels.noNewItems; + if (append && !result.items.length && searchCursor) return labels.noNewItems; if (!allItems.length) return labels.noResults; if (result.matched) return labels.showingOfMatched(allItems.length, result.matched); return labels.showing(allItems.length); @@ -888,7 +888,7 @@ function buildPanel(container: HTMLElement): () => void { additional: parseAdditionalParams(), limit: 20, next: append ? nextPage : undefined, - cursor: append ? walkCursor : undefined, + cursor: append ? searchCursor : undefined, signal: controller.signal, }; const response = connection.isApi @@ -897,13 +897,13 @@ function buildPanel(container: HTMLElement): () => void { if (generation !== searchGeneration) return; allItems = append ? [...allItems, ...response.items] : response.items; nextPage = response.next; - walkCursor = response.cursor; + searchCursor = response.cursor; // A fresh search invalidates the selection; "Load more" keeps it. if (!append) selectedItemId = null; renderItems(); showFootprints(allItems); applySelection(false); - loadMore.hidden = !nextPage && !walkCursor; + loadMore.hidden = !nextPage && !searchCursor; clearResultsButton.disabled = allItems.length === 0; setStatus(searchStatus(append, response)); } catch (error) { diff --git a/packages/plugins/src/plugins/stac-api.ts b/packages/plugins/src/plugins/stac-api.ts index 090e2ef771..8c8cd99d1c 100644 --- a/packages/plugins/src/plugins/stac-api.ts +++ b/packages/plugins/src/plugins/stac-api.ts @@ -60,7 +60,7 @@ export interface StacConnection { } /** A walk in progress; hand it back to continue. Mutated in place rather than copied. */ -export interface StacWalkCursor { +export interface StacSearchCursor { items: Unread[]; folders: Unread[]; visited: Set; @@ -72,7 +72,7 @@ export interface StacSearchOptions { bbox?: [number, number, number, number]; datetime?: string; collections?: string[]; - cursor?: StacWalkCursor; + cursor?: StacSearchCursor; /** Additional STAC API Item Search members such as query, filter, sortby, or fields. */ additional?: Record; limit?: number; @@ -89,7 +89,7 @@ export interface StacNextPage { export interface StacSearchResult { items: StacItem[]; next?: StacNextPage; - cursor?: StacWalkCursor; + cursor?: StacSearchCursor; matched?: number; } From 62c235b98119061050cf1c7a17c1fc0fb7a0ff57 Mon Sep 17 00:00:00 2001 From: Clinton Lunn Date: Fri, 14 Aug 2026 20:30:00 -0600 Subject: [PATCH 4/6] fix(stac): retry a failed catalog read instead of dropping it from the search --- packages/plugins/src/plugins/stac-api.ts | 40 ++++++++++++++----- tests/stac-api.test.ts | 50 ++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/packages/plugins/src/plugins/stac-api.ts b/packages/plugins/src/plugins/stac-api.ts index 8c8cd99d1c..94dd4ddc15 100644 --- a/packages/plugins/src/plugins/stac-api.ts +++ b/packages/plugins/src/plugins/stac-api.ts @@ -329,7 +329,10 @@ function inTime(item: StacItem, interval?: string): boolean { /** Searches a static catalog by following child/item links, with a hard safety cap. */ /** Queued but unread; the root arrives already read. */ -type Unread = { url: string; document?: Record }; +type Unread = { url: string; document?: Record; retried?: boolean }; + +/** A read about to happen, and the queue it came out of, so a failure can go back there. */ +type Pending = { entry: Unread; from: Unread[] }; export async function searchStaticStac( connection: StacConnection, @@ -356,25 +359,42 @@ export async function searchStaticStac( return inTime(item, options.datetime); }; - const takeBatch = (): Unread[] => { + const takeBatch = (): Pending[] => { const room = Math.min( STATIC_SEARCH_CONCURRENCY, limit - found.length, STATIC_SEARCH_READS_PER_PAGE - reads, ); - const batch: Unread[] = []; + const batch: Pending[] = []; while (batch.length < room && (walk.items.length || walk.folders.length)) { - const entry = (walk.items.length ? walk.items : walk.folders).shift()!; + const from = walk.items.length ? walk.items : walk.folders; + const entry = from.shift()!; if (walk.visited.has(entry.url)) continue; walk.visited.add(entry.url); - batch.push(entry); + batch.push({ entry, from }); } return batch; }; - const read = async (entry: Unread): Promise> => - entry.document ?? - fetchJson>(entry.url, { signal: options.signal }, fetcher); + /** + * A batch leaves its queue before the requests go out, so a failed read has to put the entry + * back or it is lost, and a folder takes its subtree with it. Twice failed is dropped. + */ + const read = async ({ entry, from }: Pending): Promise | undefined> => { + if (entry.document) return entry.document; + try { + return await fetchJson>( + entry.url, + { signal: options.signal }, + fetcher, + ); + } catch { + if (entry.retried) return undefined; + walk.visited.delete(entry.url); + from.unshift({ url: entry.url, retried: true }); + return undefined; + } + }; const collect = (document: Record, url: string): void => { if (document.type !== "Feature") { @@ -394,7 +414,9 @@ export async function searchStaticStac( if (!batch.length) break; reads += batch.length; const documents = await Promise.all(batch.map(read)); - documents.forEach((document, index) => collect(document, batch[index].url)); + documents.forEach((document, index) => { + if (document) collect(document, batch[index].entry.url); + }); } const offset = walk.offset + found.length; diff --git a/tests/stac-api.test.ts b/tests/stac-api.test.ts index 010417a80d..6c8e1c1456 100644 --- a/tests/stac-api.test.ts +++ b/tests/stac-api.test.ts @@ -382,6 +382,56 @@ test("a page stops reading at its budget rather than crawling the whole catalog" assert.ok(result.cursor, "and report that the walk is unfinished"); }); +test("a read that fails once is retried rather than dropped from the search", async () => { + // The batch leaves the queue before its requests go out, so a failure that took the batch with + // it would strand every document in it — and any folder among them, its whole subtree. + let failures = 0; + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: [ + { rel: "item", href: "./flaky.json" }, + { rel: "item", href: "./steady.json" }, + ], + }, + }; + for (const id of ["flaky", "steady"]) { + docs[`https://example.com/stac/${id}.json`] = { + type: "Feature", + id, + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }; + } + const fetcher = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("flaky.json") && failures === 0) { + failures += 1; + throw new Error("network"); + } + return jsonResponse(docs[url]); + }) as typeof fetch; + + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }; + + const first = await searchStaticStac(connection, { limit: 20 }, fetcher); + const ids = [...first.items.map((item) => item.id)]; + if (first.cursor) { + const second = await searchStaticStac(connection, { limit: 20, cursor: first.cursor }, fetcher); + ids.push(...second.items.map((item) => item.id)); + } + assert.deepEqual(ids.sort(), ["flaky", "steady"]); +}); + test("asset and bbox helpers recognize common STAC data", () => { assert.equal(isVisualizableAsset({ href: "https://example.com/a.TIF?download=1" }), true); assert.equal(isVisualizableAsset({ href: "https://example.com/data.bin" }), false); From 0444c75d68e97c9df4afc3f4831b13390905c0a9 Mon Sep 17 00:00:00 2001 From: Clinton Lunn Date: Fri, 14 Aug 2026 20:30:00 -0600 Subject: [PATCH 5/6] fix(stac): don't report an unfinished static search as having no results --- .../src/components/layout/TopToolbar.tsx | 2 +- .../geolibre-desktop/src/i18n/locales/en.json | 2 +- packages/plugins/src/plugins/maplibre-stac.ts | 10 ++-- tests/stac-api.test.ts | 51 +++++++++++++++++++ 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx b/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx index 0b7660c556..be165c0e03 100644 --- a/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx +++ b/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx @@ -906,7 +906,7 @@ export function TopToolbar({ resultsCleared: t("stacPlugin.resultsCleared"), searching: t("stacPlugin.searching"), loadingMore: t("stacPlugin.loadingMore"), - noNewItems: t("stacPlugin.noNewItems"), + noMatchesHere: t("stacPlugin.noMatchesHere"), noResults: t("stacPlugin.noResults"), searchFailed: t("stacPlugin.searchFailed"), showing: (count) => t("stacPlugin.showing", { count }), diff --git a/apps/geolibre-desktop/src/i18n/locales/en.json b/apps/geolibre-desktop/src/i18n/locales/en.json index 8c5975dddc..b82ee840e4 100644 --- a/apps/geolibre-desktop/src/i18n/locales/en.json +++ b/apps/geolibre-desktop/src/i18n/locales/en.json @@ -3399,7 +3399,7 @@ "resultsCleared": "Search results cleared.", "searching": "Searching STAC items…", "loadingMore": "Loading more items…", - "noNewItems": "No new items in that part of the catalog. Load more to keep searching.", + "noMatchesHere": "Nothing matched in that part of the catalog. Load more to keep searching.", "noResults": "No STAC items matched these filters.", "searchFailed": "STAC search failed", "showing": "Showing {{count}} items.", diff --git a/packages/plugins/src/plugins/maplibre-stac.ts b/packages/plugins/src/plugins/maplibre-stac.ts index 6ce2f51c8e..3c4b32ad69 100644 --- a/packages/plugins/src/plugins/maplibre-stac.ts +++ b/packages/plugins/src/plugins/maplibre-stac.ts @@ -113,7 +113,7 @@ export interface StacLabels { resultsCleared: string; searching: string; loadingMore: string; - noNewItems: string; + noMatchesHere: string; noResults: string; searchFailed: string; loadMore: string; @@ -181,7 +181,7 @@ let labels: StacLabels = { resultsCleared: "Search results cleared.", searching: "Searching STAC items…", loadingMore: "Loading more items…", - noNewItems: "No new items in that part of the catalog. Load more to keep searching.", + noMatchesHere: "Nothing matched in that part of the catalog. Load more to keep searching.", noResults: "No STAC items matched these filters.", searchFailed: "STAC search failed", loadMore: "Load more", @@ -861,8 +861,8 @@ function buildPanel(container: HTMLElement): () => void { return parsed as Record; }; - const searchStatus = (append: boolean, result: StacSearchResult): string => { - if (append && !result.items.length && searchCursor) return labels.noNewItems; + const searchStatus = (result: StacSearchResult): string => { + if (!result.items.length && result.cursor) return labels.noMatchesHere; if (!allItems.length) return labels.noResults; if (result.matched) return labels.showingOfMatched(allItems.length, result.matched); return labels.showing(allItems.length); @@ -905,7 +905,7 @@ function buildPanel(container: HTMLElement): () => void { applySelection(false); loadMore.hidden = !nextPage && !searchCursor; clearResultsButton.disabled = allItems.length === 0; - setStatus(searchStatus(append, response)); + setStatus(searchStatus(response)); } catch (error) { setStatus(error instanceof Error ? error.message : labels.searchFailed, true); } finally { diff --git a/tests/stac-api.test.ts b/tests/stac-api.test.ts index 6c8e1c1456..ae4ca85123 100644 --- a/tests/stac-api.test.ts +++ b/tests/stac-api.test.ts @@ -432,6 +432,57 @@ test("a read that fails once is retried rather than dropped from the search", as assert.deepEqual(ids.sort(), ["flaky", "steady"]); }); +test("a page that runs out of reads before matching anything returns a cursor, not a total", async () => { + // The panel says "no results" off a finished empty page, so an unfinished one must not look + // finished: the match here sits past the first page's read budget. + const root = { + type: "Catalog", + links: Array.from({ length: 400 }, (_, index) => ({ + rel: "child", + href: `./child-${index}.json`, + })), + }; + const fetcher = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("child-399.json")) { + return jsonResponse({ type: "Catalog", links: [{ rel: "item", href: "./deep.json" }] }); + } + if (url.endsWith("deep.json")) { + return jsonResponse({ + type: "Feature", + id: "deep", + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }); + } + return jsonResponse({ type: "Catalog", links: [] }); + }) as typeof fetch; + + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root, + }; + + const first = await searchStaticStac(connection, { limit: 20 }, fetcher); + assert.deepEqual(first.items, []); + assert.ok(first.cursor, "an unfinished walk must hand back a cursor"); + assert.equal(first.matched, undefined, "an unfinished walk has no total to report"); + + const second = await searchStaticStac(connection, { limit: 20, cursor: first.cursor }, fetcher); + assert.deepEqual( + second.items.map((item) => item.id), + ["deep"], + ); + assert.equal(second.cursor, undefined); + assert.equal(second.matched, 1); +}); + test("asset and bbox helpers recognize common STAC data", () => { assert.equal(isVisualizableAsset({ href: "https://example.com/a.TIF?download=1" }), true); assert.equal(isVisualizableAsset({ href: "https://example.com/data.bin" }), false); From d5a1e358c0438088362f4fe061580f067d2a3885 Mon Sep 17 00:00:00 2001 From: Clinton Lunn Date: Fri, 14 Aug 2026 22:56:37 -0600 Subject: [PATCH 6/6] fix(stac): withhold the total when a document could not be read --- packages/plugins/src/plugins/stac-api.ts | 11 ++++-- tests/stac-api.test.ts | 44 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/plugins/src/plugins/stac-api.ts b/packages/plugins/src/plugins/stac-api.ts index 94dd4ddc15..20688b88a2 100644 --- a/packages/plugins/src/plugins/stac-api.ts +++ b/packages/plugins/src/plugins/stac-api.ts @@ -66,6 +66,8 @@ export interface StacSearchCursor { visited: Set; /** Items already delivered, so the last page can report a real total. */ offset: number; + /** Documents given up on; with any of these the catalog was not fully read. */ + dropped: number; } export interface StacSearchOptions { @@ -344,6 +346,7 @@ export async function searchStaticStac( folders: [{ url: connection.url, document: connection.root }], visited: new Set(), offset: 0, + dropped: 0, }; const found: StacItem[] = []; const limit = Math.max(1, Math.min(options.limit ?? 20, 100)); @@ -389,7 +392,10 @@ export async function searchStaticStac( fetcher, ); } catch { - if (entry.retried) return undefined; + if (entry.retried) { + walk.dropped += 1; + return undefined; + } walk.visited.delete(entry.url); from.unshift({ url: entry.url, retried: true }); return undefined; @@ -422,7 +428,8 @@ export async function searchStaticStac( const offset = walk.offset + found.length; const done = !walk.items.length && !walk.folders.length; // Counting every page, not the last: the panel accumulates, so a page total reads "25 of 5". - if (done) return { items: found, matched: offset }; + // A dropped document leaves part of the catalog unread, so the count is no longer a total. + if (done) return { items: found, matched: walk.dropped ? undefined : offset }; walk.offset = offset; return { items: found, cursor: walk }; } diff --git a/tests/stac-api.test.ts b/tests/stac-api.test.ts index ae4ca85123..9b78e2ee6a 100644 --- a/tests/stac-api.test.ts +++ b/tests/stac-api.test.ts @@ -483,6 +483,50 @@ test("a page that runs out of reads before matching anything returns a cursor, n assert.equal(second.matched, 1); }); +test("a document that never reads leaves the search without a total", async () => { + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: [ + { rel: "item", href: "./good.json" }, + { rel: "child", href: "./dead.json" }, + ], + }, + "https://example.com/stac/good.json": { + type: "Feature", + id: "good", + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }, + }; + const fetcher = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("dead.json")) throw new Error("gone"); + return jsonResponse(docs[url]); + }) as typeof fetch; + + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }; + + let result = await searchStaticStac(connection, { limit: 20 }, fetcher); + const ids = result.items.map((item) => item.id); + while (result.cursor) { + result = await searchStaticStac(connection, { limit: 20, cursor: result.cursor }, fetcher); + ids.push(...result.items.map((item) => item.id)); + } + assert.deepEqual(ids, ["good"]); + // The dead child's subtree went unread, so "1 of 1" would overstate what was searched. + assert.equal(result.matched, undefined); +}); + test("asset and bbox helpers recognize common STAC data", () => { assert.equal(isVisualizableAsset({ href: "https://example.com/a.TIF?download=1" }), true); assert.equal(isVisualizableAsset({ href: "https://example.com/data.bin" }), false);