diff --git a/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx b/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx index 33106940b9..be165c0e03 100644 --- a/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx +++ b/apps/geolibre-desktop/src/components/layout/TopToolbar.tsx @@ -906,6 +906,7 @@ export function TopToolbar({ resultsCleared: t("stacPlugin.resultsCleared"), searching: t("stacPlugin.searching"), loadingMore: t("stacPlugin.loadingMore"), + noMatchesHere: t("stacPlugin.noMatchesHere"), noResults: t("stacPlugin.noResults"), searchFailed: t("stacPlugin.searchFailed"), showing: (count) => t("stacPlugin.showing", { count }), diff --git a/apps/geolibre-desktop/src/i18n/locales/en.json b/apps/geolibre-desktop/src/i18n/locales/en.json index 6b68bf8393..d03d8270eb 100644 --- a/apps/geolibre-desktop/src/i18n/locales/en.json +++ b/apps/geolibre-desktop/src/i18n/locales/en.json @@ -3399,6 +3399,7 @@ "resultsCleared": "Search results cleared.", "searching": "Searching STAC items…", "loadingMore": "Loading more items…", + "noMatchesHere": "Nothing matched in that part of the catalog. Load more to keep searching.", "noResults": "No STAC items matched these filters.", "searchFailed": "STAC search failed", "showing": "Showing {{count}} items.", diff --git a/packages/plugins/src/plugins/maplibre-stac.ts b/packages/plugins/src/plugins/maplibre-stac.ts index 829e3c4d2b..3c4b32ad69 100644 --- a/packages/plugins/src/plugins/maplibre-stac.ts +++ b/packages/plugins/src/plugins/maplibre-stac.ts @@ -15,6 +15,8 @@ import { type StacIndexCatalog, type StacItem, type StacNextPage, + type StacSearchResult, + type StacSearchCursor, } from "./stac-api"; export const STAC_PLUGIN_ID = "geolibre-stac-catalogs"; @@ -111,6 +113,7 @@ export interface StacLabels { resultsCleared: string; searching: string; loadingMore: string; + noMatchesHere: string; noResults: string; searchFailed: string; loadMore: string; @@ -178,6 +181,7 @@ let labels: StacLabels = { resultsCleared: "Search results cleared.", searching: "Searching STAC items…", loadingMore: "Loading more items…", + noMatchesHere: "Nothing matched in that part of the catalog. Load more to keep searching.", noResults: "No STAC items matched these filters.", searchFailed: "STAC search failed", loadMore: "Load more", @@ -245,7 +249,8 @@ const style = { // the controls above it scroll as a group rather than pushing it off-panel. results: "display:flex;flex:1 1 auto;min-height:150px;overflow:auto;flex-direction:column;gap:7px;", - controls: "display:flex;flex-direction:column;gap:10px;flex:0 1 auto;min-height:0;overflow:auto;", + controls: + "display:flex;flex-direction:column;gap:10px;flex:0 1 auto;min-height:180px;overflow:auto;", card: "display:flex;flex-direction:column;gap:5px;padding:8px;border:1px solid hsl(var(--border));" + "border-radius:7px;background:hsl(var(--muted));", @@ -661,6 +666,7 @@ function buildPanel(container: HTMLElement): () => void { let filtered: StacIndexCatalog[] = []; let connection: StacConnection | null = null; let nextPage: StacNextPage | undefined; + let searchCursor: StacSearchCursor | undefined; let allItems: StacItem[] = []; let searchGeneration = 0; let cancelDraw: (() => void) | null = null; @@ -714,6 +720,7 @@ function buildPanel(container: HTMLElement): () => void { searchGeneration += 1; allItems = []; nextPage = undefined; + searchCursor = undefined; results.innerHTML = ""; cardsByItemId.clear(); selectItem(null, false); @@ -854,6 +861,13 @@ function buildPanel(container: HTMLElement): () => void { return parsed as Record; }; + const searchStatus = (result: StacSearchResult): string => { + if (!result.items.length && result.cursor) return labels.noMatchesHere; + if (!allItems.length) return labels.noResults; + if (result.matched) return labels.showingOfMatched(allItems.length, result.matched); + return labels.showing(allItems.length); + }; + const runSearch = async (append: boolean): Promise => { if (!connection) return; const generation = ++searchGeneration; @@ -874,6 +888,7 @@ function buildPanel(container: HTMLElement): () => void { additional: parseAdditionalParams(), limit: 20, next: append ? nextPage : undefined, + cursor: append ? searchCursor : undefined, signal: controller.signal, }; const response = connection.isApi @@ -882,20 +897,15 @@ function buildPanel(container: HTMLElement): () => void { if (generation !== searchGeneration) return; allItems = append ? [...allItems, ...response.items] : response.items; nextPage = response.next; + searchCursor = response.cursor; // A fresh search invalidates the selection; "Load more" keeps it. if (!append) selectedItemId = null; renderItems(); showFootprints(allItems); applySelection(false); - loadMore.hidden = !nextPage; + loadMore.hidden = !nextPage && !searchCursor; clearResultsButton.disabled = allItems.length === 0; - setStatus( - allItems.length - ? response.matched - ? labels.showingOfMatched(allItems.length, response.matched) - : labels.showing(allItems.length) - : labels.noResults, - ); + setStatus(searchStatus(response)); } catch (error) { setStatus(error instanceof Error ? error.message : labels.searchFailed, true); } finally { diff --git a/packages/plugins/src/plugins/stac-api.ts b/packages/plugins/src/plugins/stac-api.ts index 3510495ebe..20688b88a2 100644 --- a/packages/plugins/src/plugins/stac-api.ts +++ b/packages/plugins/src/plugins/stac-api.ts @@ -1,6 +1,9 @@ import type { BBox, Feature, Geometry } from "geojson"; export const STAC_INDEX_CATALOGS_URL = "https://stacindex.org/api/catalogs"; +// No item-search endpoint to ask, so a page is however much of the tree the walk covers. +const STATIC_SEARCH_READS_PER_PAGE = 300; +const STATIC_SEARCH_CONCURRENCY = 12; export interface StacIndexCatalog { id: number; @@ -56,10 +59,22 @@ export interface StacConnection { root: Record; } +/** A walk in progress; hand it back to continue. Mutated in place rather than copied. */ +export interface StacSearchCursor { + items: Unread[]; + folders: Unread[]; + visited: Set; + /** Items already delivered, so the last page can report a real total. */ + offset: number; + /** Documents given up on; with any of these the catalog was not fully read. */ + dropped: number; +} + export interface StacSearchOptions { bbox?: [number, number, number, number]; datetime?: string; collections?: string[]; + cursor?: StacSearchCursor; /** Additional STAC API Item Search members such as query, filter, sortby, or fields. */ additional?: Record; limit?: number; @@ -76,6 +91,7 @@ export interface StacNextPage { export interface StacSearchResult { items: StacItem[]; next?: StacNextPage; + cursor?: StacSearchCursor; matched?: number; } @@ -109,7 +125,7 @@ export function browserAssetHref(href: string, base: string): string { return `https://${bucket}.s3.amazonaws.com${url.pathname}${url.search}${url.hash}`; } -async function fetchJson(url: string, init: RequestInit, fetcher: FetchLike): Promise { +async function fetchJson(url: string, init: RequestInit, fetcher: FetchLike): Promise { const response = await fetcher(url, { ...init, headers: { Accept: "application/geo+json, application/json", ...init.headers }, @@ -148,6 +164,13 @@ function linksOf(value: unknown, base: string): StacLink[] { }); } +function isStacItem(value: unknown): value is StacItem { + if (typeof value !== "object" || value === null) return false; + return ( + "id" in value && typeof value.id === "string" && "assets" in value && Boolean(value.assets) + ); +} + function normalizeItem(item: StacItem, base: string): StacItem { const assets = Object.fromEntries( Object.entries(item.assets ?? {}).flatMap(([key, asset]) => { @@ -169,9 +192,9 @@ export async function connectStac( ): Promise { if (!httpUrl(inputUrl)) throw new Error("Enter a valid HTTP or HTTPS STAC URL"); const url = new URL(inputUrl).href; - const raw = await fetchJson(url, { signal }, fetcher); - if (!raw || typeof raw !== "object") throw new Error("The URL did not return a STAC document"); - const root = raw as Record; + const root = await fetchJson>(url, { signal }, fetcher); + if (typeof root !== "object" || root === null) + throw new Error("The URL did not return a STAC document"); const links = linksOf(root.links, url); const conforms = Array.isArray(root.conformsTo) ? root.conformsTo.map(String) : []; const searchLink = links.find((link) => link.rel === "search"); @@ -186,9 +209,11 @@ export async function connectStac( ); if (collectionsLink) { try { - const data = (await fetchJson(collectionsLink.href, { signal }, fetcher)) as { - collections?: StacCollection[]; - }; + const data = await fetchJson<{ collections?: StacCollection[] }>( + collectionsLink.href, + { signal }, + fetcher, + ); if (Array.isArray(data.collections)) collections = data.collections; } catch { // Collection discovery is helpful but not required for item search. @@ -213,15 +238,7 @@ function parseItems(raw: unknown, responseUrl: string): StacSearchResult { throw new Error("The STAC server returned invalid search data"); const data = raw as Record; const features = Array.isArray(data.features) ? data.features : []; - const items = features - .filter( - (feature): feature is StacItem => - Boolean(feature) && - typeof feature === "object" && - typeof (feature as StacItem).id === "string" && - Boolean((feature as StacItem).assets), - ) - .map((item) => normalizeItem(item, responseUrl)); + const items = features.filter(isStacItem).map((item) => normalizeItem(item, responseUrl)); const nextLink = linksOf(data.links, responseUrl).find((link) => link.rel === "next"); const context = data.context as { matched?: unknown } | undefined; const numberMatched = data.numberMatched; @@ -313,46 +330,108 @@ function inTime(item: StacItem, interval?: string): boolean { } /** Searches a static catalog by following child/item links, with a hard safety cap. */ +/** Queued but unread; the root arrives already read. */ +type Unread = { url: string; document?: Record; retried?: boolean }; + +/** A read about to happen, and the queue it came out of, so a failure can go back there. */ +type Pending = { entry: Unread; from: Unread[] }; + export async function searchStaticStac( connection: StacConnection, options: StacSearchOptions, fetcher: FetchLike = fetch, ): Promise { - const queue: Array<{ url: string; document?: Record }> = [ - { url: connection.url, document: connection.root }, - ]; - const visited = new Set(); - const items: StacItem[] = []; + const walk = options.cursor ?? { + items: [], + folders: [{ url: connection.url, document: connection.root }], + visited: new Set(), + offset: 0, + dropped: 0, + }; + const found: StacItem[] = []; const limit = Math.max(1, Math.min(options.limit ?? 20, 100)); - while (queue.length && visited.size < 300 && items.length < limit) { - const current = queue.shift()!; - if (visited.has(current.url)) continue; - visited.add(current.url); - const document = - current.document ?? - ((await fetchJson(current.url, { signal: options.signal }, fetcher)) as Record< - string, - unknown - >); - if (document.type === "Feature") { - const item = normalizeItem(document as unknown as StacItem, current.url); - // itemBbox flattens 3D (6-element) bboxes; item.bbox[2]/[3] would be minZ/maxX there. - const bbox = itemBbox(item); - if ( - (!options.collections?.length || - (item.collection && options.collections.includes(item.collection))) && - (!options.bbox || (bbox && intersects(bbox, options.bbox))) && - inTime(item, options.datetime) - ) { - items.push(item); + let reads = 0; + + const accepts = (item: StacItem): boolean => { + // itemBbox flattens 3D (6-element) bboxes; item.bbox[2]/[3] would be minZ/maxX there. + const bbox = itemBbox(item); + if (options.collections?.length && !options.collections.includes(item.collection ?? "")) { + return false; + } + if (options.bbox && !(bbox && intersects(bbox, options.bbox))) return false; + return inTime(item, options.datetime); + }; + + const takeBatch = (): Pending[] => { + const room = Math.min( + STATIC_SEARCH_CONCURRENCY, + limit - found.length, + STATIC_SEARCH_READS_PER_PAGE - reads, + ); + const batch: Pending[] = []; + while (batch.length < room && (walk.items.length || walk.folders.length)) { + const from = walk.items.length ? walk.items : walk.folders; + const entry = from.shift()!; + if (walk.visited.has(entry.url)) continue; + walk.visited.add(entry.url); + batch.push({ entry, from }); + } + return batch; + }; + + /** + * A batch leaves its queue before the requests go out, so a failed read has to put the entry + * back or it is lost, and a folder takes its subtree with it. Twice failed is dropped. + */ + const read = async ({ entry, from }: Pending): Promise | undefined> => { + if (entry.document) return entry.document; + try { + return await fetchJson>( + entry.url, + { signal: options.signal }, + fetcher, + ); + } catch { + if (entry.retried) { + walk.dropped += 1; + return undefined; } - continue; + walk.visited.delete(entry.url); + from.unshift({ url: entry.url, retried: true }); + return undefined; } - for (const link of linksOf(document.links, current.url)) { - if (link.rel === "item" || link.rel === "child") queue.push({ url: link.href }); + }; + + const collect = (document: Record, url: string): void => { + if (document.type !== "Feature") { + for (const link of linksOf(document.links, url)) { + if (link.rel === "item") walk.items.push({ url: link.href }); + else if (link.rel === "child") walk.folders.push({ url: link.href }); + } + return; } + if (!isStacItem(document)) return; + const item = normalizeItem(document, url); + if (accepts(item)) found.push(item); + }; + + while (found.length < limit && reads < STATIC_SEARCH_READS_PER_PAGE) { + const batch = takeBatch(); + if (!batch.length) break; + reads += batch.length; + const documents = await Promise.all(batch.map(read)); + documents.forEach((document, index) => { + if (document) collect(document, batch[index].entry.url); + }); } - return { items, matched: items.length }; + + const offset = walk.offset + found.length; + const done = !walk.items.length && !walk.folders.length; + // Counting every page, not the last: the panel accumulates, so a page total reads "25 of 5". + // A dropped document leaves part of the catalog unread, so the count is no longer a total. + if (done) return { items: found, matched: walk.dropped ? undefined : offset }; + walk.offset = offset; + return { items: found, cursor: walk }; } export function itemBbox(item: StacItem): [number, number, number, number] | undefined { diff --git a/tests/stac-api.test.ts b/tests/stac-api.test.ts index 5d46289578..9b78e2ee6a 100644 --- a/tests/stac-api.test.ts +++ b/tests/stac-api.test.ts @@ -241,6 +241,292 @@ test("searchStaticStac traverses child and item links and applies filters", asyn ); }); +test("searchStaticStac pages through a catalog holding more items than one page fits", async () => { + const total = 25; + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: Array.from({ length: total }, (_value, index) => ({ + rel: "item", + href: `./item${index}.json`, + })), + }, + }; + for (let index = 0; index < total; index += 1) { + docs[`https://example.com/stac/item${index}.json`] = { + type: "Feature", + id: `item${index}`, + collection: "many", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }; + } + const fetcher = (async (input: RequestInfo | URL) => + jsonResponse(docs[String(input)])) as typeof fetch; + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }; + + const first = await searchStaticStac(connection, { limit: 10 }, fetcher); + assert.equal(first.items.length, 10); + assert.ok(first.cursor, "a walk with documents left over reports where it stopped"); + assert.equal(first.matched, undefined); + + const second = await searchStaticStac(connection, { limit: 10, cursor: first.cursor }, fetcher); + assert.equal(second.items.length, 10); + assert.deepEqual( + second.items.map((item) => item.id).filter((id) => first.items.some((seen) => seen.id === id)), + [], + "a resumed page repeats nothing from the page before it", + ); + + const third = await searchStaticStac(connection, { limit: 10, cursor: second.cursor }, fetcher); + assert.equal(third.items.length, 5); + assert.equal(third.cursor, undefined, "the walk is done, so there is nothing to resume"); + assert.equal(third.matched, 25); +}); + +test("searchStaticStac reads items before folders, so a page is not spent on structure", async () => { + // Items one folder down, behind a hundred empty ones. Discovery order spends the page on + // folders and returns nothing. + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: [ + { rel: "child", href: "./has-items.json" }, + ...Array.from({ length: 100 }, (_value, index) => ({ + rel: "child", + href: `./empty${index}.json`, + })), + ], + }, + "https://example.com/stac/has-items.json": { + type: "Catalog", + links: Array.from({ length: 3 }, (_value, index) => ({ + rel: "item", + href: `./item${index}.json`, + })), + }, + }; + for (let index = 0; index < 100; index += 1) { + docs[`https://example.com/stac/empty${index}.json`] = { type: "Catalog", links: [] }; + } + for (let index = 0; index < 3; index += 1) { + docs[`https://example.com/stac/item${index}.json`] = { + type: "Feature", + id: `item${index}`, + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }; + } + let reads = 0; + const fetcher = (async (input: RequestInfo | URL) => { + reads += 1; + return jsonResponse(docs[String(input)]); + }) as typeof fetch; + + const result = await searchStaticStac( + { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }, + { limit: 3 }, + fetcher, + ); + + assert.equal(result.items.length, 3); + // Discovery order costs a hundred more. + assert.ok(reads < 30, `expected the items to be reached quickly, took ${reads} reads`); +}); + +test("a page stops reading at its budget rather than crawling the whole catalog", async () => { + // No items anywhere, so only the budget can end the page. + let reads = 0; + const fetcher = (async () => { + reads += 1; + return jsonResponse({ + type: "Catalog", + links: [ + { rel: "child", href: `./${reads}-a.json` }, + { rel: "child", href: `./${reads}-b.json` }, + ], + }); + }) as typeof fetch; + + const result = await searchStaticStac( + { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: { type: "Catalog", links: [{ rel: "child", href: "./a.json" }] }, + }, + { limit: 20 }, + fetcher, + ); + + assert.deepEqual(result.items, []); + assert.ok(reads <= 300, `a page must stop at its budget, read ${reads}`); + assert.ok(result.cursor, "and report that the walk is unfinished"); +}); + +test("a read that fails once is retried rather than dropped from the search", async () => { + // The batch leaves the queue before its requests go out, so a failure that took the batch with + // it would strand every document in it — and any folder among them, its whole subtree. + let failures = 0; + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: [ + { rel: "item", href: "./flaky.json" }, + { rel: "item", href: "./steady.json" }, + ], + }, + }; + for (const id of ["flaky", "steady"]) { + docs[`https://example.com/stac/${id}.json`] = { + type: "Feature", + id, + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }; + } + const fetcher = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("flaky.json") && failures === 0) { + failures += 1; + throw new Error("network"); + } + return jsonResponse(docs[url]); + }) as typeof fetch; + + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }; + + const first = await searchStaticStac(connection, { limit: 20 }, fetcher); + const ids = [...first.items.map((item) => item.id)]; + if (first.cursor) { + const second = await searchStaticStac(connection, { limit: 20, cursor: first.cursor }, fetcher); + ids.push(...second.items.map((item) => item.id)); + } + assert.deepEqual(ids.sort(), ["flaky", "steady"]); +}); + +test("a page that runs out of reads before matching anything returns a cursor, not a total", async () => { + // The panel says "no results" off a finished empty page, so an unfinished one must not look + // finished: the match here sits past the first page's read budget. + const root = { + type: "Catalog", + links: Array.from({ length: 400 }, (_, index) => ({ + rel: "child", + href: `./child-${index}.json`, + })), + }; + const fetcher = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("child-399.json")) { + return jsonResponse({ type: "Catalog", links: [{ rel: "item", href: "./deep.json" }] }); + } + if (url.endsWith("deep.json")) { + return jsonResponse({ + type: "Feature", + id: "deep", + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }); + } + return jsonResponse({ type: "Catalog", links: [] }); + }) as typeof fetch; + + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root, + }; + + const first = await searchStaticStac(connection, { limit: 20 }, fetcher); + assert.deepEqual(first.items, []); + assert.ok(first.cursor, "an unfinished walk must hand back a cursor"); + assert.equal(first.matched, undefined, "an unfinished walk has no total to report"); + + const second = await searchStaticStac(connection, { limit: 20, cursor: first.cursor }, fetcher); + assert.deepEqual( + second.items.map((item) => item.id), + ["deep"], + ); + assert.equal(second.cursor, undefined); + assert.equal(second.matched, 1); +}); + +test("a document that never reads leaves the search without a total", async () => { + const docs: Record = { + "https://example.com/stac/catalog.json": { + type: "Catalog", + links: [ + { rel: "item", href: "./good.json" }, + { rel: "child", href: "./dead.json" }, + ], + }, + "https://example.com/stac/good.json": { + type: "Feature", + id: "good", + collection: "c", + bbox: [0, 0, 1, 1], + geometry: null, + properties: { datetime: "2024-05-01T00:00:00Z" }, + assets: {}, + }, + }; + const fetcher = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("dead.json")) throw new Error("gone"); + return jsonResponse(docs[url]); + }) as typeof fetch; + + const connection = { + url: "https://example.com/stac/catalog.json", + title: "Static", + isApi: false, + collections: [], + root: docs["https://example.com/stac/catalog.json"] as Record, + }; + + let result = await searchStaticStac(connection, { limit: 20 }, fetcher); + const ids = result.items.map((item) => item.id); + while (result.cursor) { + result = await searchStaticStac(connection, { limit: 20, cursor: result.cursor }, fetcher); + ids.push(...result.items.map((item) => item.id)); + } + assert.deepEqual(ids, ["good"]); + // The dead child's subtree went unread, so "1 of 1" would overstate what was searched. + assert.equal(result.matched, undefined); +}); + test("asset and bbox helpers recognize common STAC data", () => { assert.equal(isVisualizableAsset({ href: "https://example.com/a.TIF?download=1" }), true); assert.equal(isVisualizableAsset({ href: "https://example.com/data.bin" }), false);