From 586f02e75f3579ea9ce53202128064b50c4a1708 Mon Sep 17 00:00:00 2001 From: giswqs Date: Tue, 18 Aug 2026 21:41:58 -0400 Subject: [PATCH 1/6] fix(chrome): find map services without broad host permissions The Chrome Web Store flags `http://*/*` and `https://*/*` for in-depth review, and those existed only so `webRequest` could watch map requests. The popup now reads each frame's Resource Timing buffer under activeTab instead, leaving the extension with activeTab and scripting alone. MapLibre fetches vector tiles from a worker, which that buffer never records, so such a tileset is recovered from the TileJSON or style the main thread did fetch; GeoLibre now accepts a vector tiles deep link carrying only a style. --- apps/geolibre-desktop/src/lib/data-url.ts | 12 +- docs/user-guide/chrome-extension.md | 2 +- extensions/geolibre-chrome/PRIVACY.md | 22 +- extensions/geolibre-chrome/README.md | 56 ++-- extensions/geolibre-chrome/STORE_LISTING.md | 47 ++- extensions/geolibre-chrome/background.mjs | 136 --------- extensions/geolibre-chrome/manifest.json | 9 +- extensions/geolibre-chrome/popup.mjs | 20 +- extensions/geolibre-chrome/scanner.mjs | 21 ++ .../geolibre-chrome/service-scanner.mjs | 153 ++++------ extensions/geolibre-chrome/url-builder.mjs | 5 +- scripts/package-chrome-extension.mjs | 1 - tests/chrome-extension.test.ts | 274 +++++++----------- tests/data-url.test.ts | 22 ++ 14 files changed, 302 insertions(+), 478 deletions(-) delete mode 100644 extensions/geolibre-chrome/background.mjs diff --git a/apps/geolibre-desktop/src/lib/data-url.ts b/apps/geolibre-desktop/src/lib/data-url.ts index 88aba58a2..8d81b746c 100644 --- a/apps/geolibre-desktop/src/lib/data-url.ts +++ b/apps/geolibre-desktop/src/lib/data-url.ts @@ -39,12 +39,18 @@ export function serviceUrlParameter(search: string): ServiceUrlParameter | null parsed && kind && TILE_TEMPLATE_KINDS.has(kind) ? parsed.replace(/%7B/gi, "{").replace(/%7D/gi, "}") : parsed; - if (!kind || !SERVICE_KINDS.has(kind) || !url) return null; + const styleUrl = httpUrl(params.get("serviceStyle")); + if (!kind || !SERVICE_KINDS.has(kind)) return null; + // A vector tileset whose style names its tiles inline is addable from that + // style alone, since the source layers and the tile template both come out of + // the same document. So a link carrying only a style is complete, and the + // dialog opens with an empty tileset field rather than not opening at all. + if (!url && !(kind === "ogc-vector-tiles" && styleUrl)) return null; return { kind, - url, + url: url ?? "", layer: params.get("serviceLayer")?.trim() || null, - styleUrl: httpUrl(params.get("serviceStyle")), + styleUrl, }; } export interface RemoteGeoJsonLayer { diff --git a/docs/user-guide/chrome-extension.md b/docs/user-guide/chrome-extension.md index cff3d8849..b7ffb9378 100644 --- a/docs/user-guide/chrome-extension.md +++ b/docs/user-guide/chrome-extension.md @@ -43,4 +43,4 @@ The extension recognizes GeoJSON and spatial JSON, GeoParquet and Parquet, PMTil GeoLibre fetches selected links directly, so the source server must allow cross-origin requests (CORS). Complete HTTP(S) URLs, including signed query parameters, are forwarded to GeoLibre. Cookies and other browser-session credentials are not forwarded, so cookie-bound or session-authenticated links might fail. Temporary `blob:` URLs cannot be transferred. -The extension requests access only to the active tab when you open it. It does not store browsing history, send analytics, or run continuously in the background. +The extension requests access only to the active tab, and only from the moment you click its icon. It holds no standing permission to any website, stores nothing, sends no analytics, and runs nothing in the background. Map services are recognized by reading back the addresses of the requests the page has already made, which the page records for itself, rather than by watching your browsing. diff --git a/extensions/geolibre-chrome/PRIVACY.md b/extensions/geolibre-chrome/PRIVACY.md index 12c2185ee..610c316fa 100644 --- a/extensions/geolibre-chrome/PRIVACY.md +++ b/extensions/geolibre-chrome/PRIVACY.md @@ -1,18 +1,20 @@ # Privacy policy — Open data in GeoLibre -Last updated: August 16, 2026 +Last updated: August 18, 2026 Open data in GeoLibre does not independently collect, retain, or sell personal information, page contents, or usage analytics. It does forward the complete dataset, service, and style URLs that the user explicitly selects, as described below. -The extension uses Chrome's `activeTab` permission to inspect links and -structured metadata on the current page only after the user clicks the -extension's toolbar icon. The extension also observes completed HTTP(S) -requests locally to identify geospatial services used by interactive maps. It -does not inspect response bodies. Detected service URLs are held in Chrome's -in-memory session storage and removed when their tab closes. +The extension uses Chrome's `activeTab` permission to inspect the current page +only after the user clicks the extension's toolbar icon. In that moment it reads +the page's links and structured metadata, and reads back the addresses of the +requests the page has already made, from the Resource Timing record each +document keeps of its own loading, to identify geospatial services used by +interactive maps. It does not inspect response bodies, and it does not observe +the network or run in the background at any other time. Nothing is stored: the +list exists only while the popup is open and is discarded when it closes. When the user chooses **Open in GeoLibre**, the complete selected HTTP(S) dataset and style URLs are placed in the query string of a new @@ -25,8 +27,7 @@ The navigation request exposes its URL and the user's IP address to GeoLibre's web-hosting infrastructure, where standard service logs may retain them. The navigation may also appear in browser history. See the current [GeoLibre privacy policy](https://geolibre.app/privacy/) for the service's data -practices. The extension itself does not persist URLs beyond the browser tab's -session. +practices. The extension itself never persists a URL. The extension does not fetch or upload the datasets. GeoLibre requests them directly from their original servers, subject to those servers' privacy @@ -35,7 +36,8 @@ from the source page are not forwarded, although credentials embedded directly in a selected URL are part of the URL and are forwarded. The extension uses no remote code, advertising, analytics, tracking pixels, -cookies, accounts, or persistent extension storage. +cookies, accounts, or extension storage of any kind. It holds no host +permissions and no permission to observe browsing. Questions may be submitted through the GeoLibre repository: . diff --git a/extensions/geolibre-chrome/README.md b/extensions/geolibre-chrome/README.md index 5be167313..d5a4a4449 100644 --- a/extensions/geolibre-chrome/README.md +++ b/extensions/geolibre-chrome/README.md @@ -1,8 +1,8 @@ # Open data in GeoLibre A Manifest V3 Chrome extension that finds supported geospatial dataset links on -the current page, observes geospatial service requests made by interactive maps, -and opens selected data in GeoLibre. +the current page, reads back the geospatial service requests its interactive +maps have already made, and opens selected data in GeoLibre. ## Install @@ -17,10 +17,10 @@ The published extension is on the Chrome Web Store: 3. Choose **Load unpacked**. 4. Select this `extensions/geolibre-chrome` directory. -The extension scans document links after you click its toolbar icon. It also -observes completed HTTP(S) requests locally so it can recognize services used -by interactive web maps. Detected service URLs remain only in session storage -for the lifetime of their tab. +Everything happens after you click the toolbar icon: the extension scans the +document's links, and reads each frame's Resource Timing buffer to recognize the +services its maps requested. It holds no permission beyond `activeTab` and +`scripting`, runs no background service worker, and stores nothing. ## Package for the Chrome Web Store @@ -49,10 +49,29 @@ virtualized, canonicalizes links to `data.source.coop`, and removes duplicate page/download links. The popup can filter discovered files by vector or raster type without changing the current selection. -The request watcher recognizes WMS, WMTS, WFS, OGC API Features, ArcGIS Feature -Services, XYZ/TMS image tiles, and PBF/MVT vector tiles. Tile requests are -collapsed into reusable `{z}/{x}/{y}` templates, and repeated requests from the -same service appear once. +## Services + +A map fetches its tiles and service documents from JavaScript, so they are never +links in the page. What the extension reads instead is `performance +.getEntriesByType("resource")`, the record of its own requests that every +document keeps, collected from the top frame and each frame below it. Recognized +are WMS, WMTS, WFS, OGC API Features, ArcGIS Feature Services, XYZ/TMS image +tiles, and PBF/MVT vector tiles. Tile requests are collapsed into reusable +`{z}/{x}/{y}` templates, and repeated requests from the same service appear once. + +Two consequences of reading the buffer rather than watching the network: + +- **Worker requests are invisible.** MapLibre and similar renderers fetch vector + tiles from a web worker, which records them in the worker's own timeline, not + the document's. Such a tileset is recovered from the metadata the main thread + *did* fetch: its TileJSON (`…/tiles.json`), or failing that the style document, + which GeoLibre can resolve a layer from on its own. A style is only offered in + its own right when no tileset from its origin was found. +- **The buffer is finite.** It holds 250 entries per document by default and + stops recording once full. A map's own early requests are normally well inside + that, but a very busy page can lose a service added late. Raising the limit + needs a `document_start` script, which needs the broad host permissions this + design exists to avoid, so the cap is accepted. A service endpoint on its own is rarely enough to add a layer, so each result also carries what the page asked that service *for*: the WMS `LAYERS` value, the @@ -70,7 +89,7 @@ After loading or reloading the unpacked extension, open one of these websites in a new tab, let its map finish drawing, then open the extension and confirm it lists the expected service. Selecting the result opens the matching GeoLibre Add Data dialog with the service URL filled in. Each row below is a live third-party -map, so what it detects is what a real page hands the request watcher. +map, so what it detects is what a real page actually requests. | Service | Website | Detected service | Layer carried over | | --- | --- | --- | --- | @@ -80,24 +99,25 @@ map, so what it detects is what a real page hands the request watcher. | OGC API Features | [pygeoapi lakes collection](https://demo.pygeoapi.io/master/collections/lakes/items?f=html) | `https://demo.pygeoapi.io/master/collections/lakes/items` | — | | ArcGIS Feature Service | [OpenLayers "Vector ESRI" example](https://openlayers.org/en/latest/examples/vector-esri.html) | The `…/FeatureServer/0` layer URL | `0` | | XYZ raster tiles | [openstreetmap.org](https://www.openstreetmap.org/) | `https://tile.openstreetmap.org/{z}/{x}/{y}.png` | — | -| Vector tiles, in an `iframe` | [MapLibre "Display a map" example](https://maplibre.org/maplibre-gl-js/docs/examples/display-a-map/) | `https://demotiles.maplibre.org/tiles/{z}/{x}/{y}.pbf` | style `…/style.json` | +| Vector tiles, in an `iframe` | [MapLibre "Display a map" example](https://maplibre.org/maplibre-gl-js/docs/examples/display-a-map/) | `https://demotiles.maplibre.org/tiles/tiles.json` | style `…/style.json` | Each row above adds a layer that draws, with no further typing: that is the bar for this table. A row that opens the dialog but leaves a required field empty is a bug, not an expected extra step. The MapLibre row is worth keeping in the set: the map runs inside an `iframe`, so -it covers services a page reaches only through an embedded frame. It also carries -a style whose glyph ranges are served as `.pbf`; those are fonts, not a tileset, -and must not be offered. +it covers services a page reaches only through an embedded frame, and it renders +through a worker, so it covers the tileset recovered from its TileJSON rather +than from a tile request. It also carries a style whose glyph ranges are served +as `.pbf`; those are fonts, not a tileset, and must not be offered. Opening a service URL directly is detected too — the response is the page, so [a WMS GetCapabilities document](https://ows.terrestris.de/osm/service?SERVICE=WMS&REQUEST=GetCapabilities) lists `https://ows.terrestris.de/osm/service`. -Navigating the same tab elsewhere replaces the list, so a page with no services -(`https://example.com`) must come up empty rather than inheriting the page before -it. +A document's buffer is its own and is discarded when the tab navigates, so a page +with no services (`https://example.com`) comes up empty rather than inheriting +the page before it. After selecting a result, add the layer in GeoLibre and verify that the browser console does not report a CORS error while fetching the service. diff --git a/extensions/geolibre-chrome/STORE_LISTING.md b/extensions/geolibre-chrome/STORE_LISTING.md index 9c2ca9eaa..e747b405e 100644 --- a/extensions/geolibre-chrome/STORE_LISTING.md +++ b/extensions/geolibre-chrome/STORE_LISTING.md @@ -13,31 +13,17 @@ Find geospatial datasets and map services on a webpage and open them in GeoLibre ## Detailed description -Open data in GeoLibre turns dataset catalogs, documentation pages, and project -websites into launch points for an interactive map. +Open data in GeoLibre turns dataset catalogs, documentation pages, and project websites into launch points for an interactive map. -Click the extension icon to find supported data links on the current page, -filter them by vector or raster type, choose the files you need, and open them -together in GeoLibre. Supported links include GeoJSON, GeoParquet, PMTiles, -Cloud-Optimized GeoTIFF, and ZIP archives containing GeoJSON. +Click the extension icon to find supported data links on the current page, filter them by vector or raster type, choose the files you need, and open them together in GeoLibre. Supported links include GeoJSON, GeoParquet, PMTiles, Cloud-Optimized GeoTIFF, and ZIP archives containing GeoJSON. -The extension also reads schema.org download metadata, understands existing -GeoLibre links, pairs matching GeoLibre style files, and discovers the complete -file inventory on virtualized Source Cooperative repository pages. +The extension also reads schema.org download metadata, understands existing GeoLibre links, pairs matching GeoLibre style files, and discovers the complete file inventory on virtualized Source Cooperative repository pages. -Interactive maps are supported too. The extension recognizes completed WMS, -WMTS, WFS, OGC API Features, ArcGIS Feature Service, XYZ/TMS, and vector-tile -requests made by the current tab. +Interactive maps are supported too. The extension recognizes the WMS, WMTS, WFS, OGC API Features, ArcGIS Feature Service, XYZ/TMS, and vector-tile requests the current page has already made. -Detected service URLs stay in temporary browser session storage only until the -tab closes. The extension runs no analytics and sends no browsing activity to -GeoLibre unless you explicitly select an item and open it. +The extension reads the page only when you click its icon, holds no standing access to any website, stores nothing, and runs nothing in the background. It runs no analytics and sends no browsing activity to GeoLibre unless you explicitly select an item and open it. -Dataset servers must allow browser access through CORS. Complete HTTP(S) URLs, -including signed query parameters, are forwarded to GeoLibre. Cookies and other -browser-session credentials are not forwarded, so cookie-bound or -session-authenticated links may fail. Temporary `blob:` links cannot be -transferred. +Dataset servers must allow browser access through CORS. Complete HTTP(S) URLs, including signed query parameters, are forwarded to GeoLibre. Cookies and other browser-session credentials are not forwarded, so cookie-bound or session-authenticated links may fail. Temporary `blob:` links cannot be transferred. ## Category @@ -49,13 +35,16 @@ English ## Permission justification -- `activeTab`: grants temporary access to the page only after the user invokes - the extension, so its dataset links can be inspected. -- `scripting`: injects the local, packaged dataset scanner into that active tab. -- `webRequest` and HTTP(S) host access: observes completed requests locally to - identify geospatial services used by interactive maps. -- `storage`: holds detected service URLs in session-only storage until their tab - closes so the popup can display them. +Each block below is self-contained and is pasted verbatim into the matching field of the Chrome Web Store dashboard's Privacy tab. Keep them in sync with `manifest.json`: a permission added there needs a justification here and in the dashboard, or the version is rejected. As of 0.3.0 the manifest requests `activeTab` and `scripting` and nothing else, so the storage, webRequest, and host-permission fields no longer appear. -The extension does not request browsing history, downloads, cookies, or remote -code. +### activeTab + +activeTab grants temporary access to the current page only after the user clicks the extension's toolbar icon. The extension uses that access to read the page's links and structured metadata and pick out geospatial datasets, such as GeoJSON, GeoParquet, PMTiles, Cloud-Optimized GeoTIFF, and ZIP archives containing GeoJSON, which it then lists in the popup for the user to choose from. Access ends when the user leaves or reloads the page, and no page content is read at any other time. + +### scripting + +scripting injects two packaged functions into the active tab when the user opens the popup. One reads the page's links and metadata to find dataset files. The other reads back the addresses of the requests the page has already made, so the map services it draws can be recognized; a map fetches those from JavaScript, so they are never links in the document. Both functions are contained in the extension package, so no remote code is involved. They run once per invocation and return their results to the popup. + +### Not requested + +The extension requests no host permissions, and no permission to watch network requests, store data, or run in the background. It does not request browsing history, downloads, cookies, tabs beyond the active one, or remote code. Answer "No, I am not using remote code": every script it runs ships inside the package. diff --git a/extensions/geolibre-chrome/background.mjs b/extensions/geolibre-chrome/background.mjs deleted file mode 100644 index b3c66e6f9..000000000 --- a/extensions/geolibre-chrome/background.mjs +++ /dev/null @@ -1,136 +0,0 @@ -import { - classifyServiceRequest, - classifyStyleRequest, - createPageScope, - createTabTaskQueue, -} from "./service-scanner.mjs"; - -const MAX_REQUESTS_PER_TAB = 100; -const enqueue = createTabTaskQueue(); -const scope = createPageScope(); -/** Style documents seen per tab, keyed by the origin that served them. */ -const stylesByTab = new Map(); -/** - * The candidate most recently written for each tab. Panning a slippy map turns - * every tile into the same candidate, so without this each one would queue a - * `storage.session` read just to discover there is nothing to write. - */ -const lastWritten = new Map(); - -function candidateKey(service) { - return [service.url, service.layer ?? "", service.styleUrl ?? ""].join("\u0000"); -} - -function forgetTab(tabId) { - stylesByTab.delete(tabId); - lastWritten.delete(tabId); -} - -function rememberStyle(tabId, style) { - let origins = stylesByTab.get(tabId); - if (!origins) { - origins = new Map(); - stylesByTab.set(tabId, origins); - } - origins.set(style.origin, style.url); -} - -function runForTab(tabId, task) { - void enqueue(tabId, task).catch((error) => { - // A failed write leaves the stored list unknown, so drop the shortcut and - // let the next matching request try again. - lastWritten.delete(tabId); - console.warn("GeoLibre could not update detected services.", error); - }); -} - -/** - * Fill in the style of vector tilesets already stored for this tab. Tiles and - * their style are separate requests and either can finish first, so a tileset - * recorded before its style arrived would otherwise stay unusable: without the - * style's source layers, Add Data cannot resolve the layer. - */ -async function applyStyleToStored(tabId, style) { - const key = `services:${tabId}`; - const stored = await chrome.storage.session.get(key); - const existing = Array.isArray(stored[key]) ? stored[key] : []; - let changed = false; - const next = existing.map((entry) => { - if (entry.format !== "Vector tiles" || entry.styleUrl) return entry; - if (new URL(entry.url).origin !== style.origin) return entry; - changed = true; - return { ...entry, styleUrl: style.url }; - }); - if (!changed) return; - lastWritten.set(tabId, candidateKey(next[0])); - await chrome.storage.session.set({ [key]: next }); -} - -// The page boundary is drawn when a navigation starts, not when it finishes, -// so the incoming page's own early requests are not retired along with the -// outgoing page's. -chrome.webRequest.onBeforeRequest.addListener( - ({ tabId, type }) => { - if (tabId >= 0 && type === "main_frame") scope.beginPage(tabId); - }, - { urls: ["http://*/*", "https://*/*"] }, -); - -chrome.webRequest.onCompleted.addListener( - ({ tabId, url, type, documentId }) => { - if (tabId < 0) return; - if (type === "main_frame") { - scope.startPage(tabId); - forgetTab(tabId); - runForTab(tabId, () => chrome.storage.session.remove(`services:${tabId}`)); - } - if (!scope.accepts(tabId, documentId)) return; - const style = classifyStyleRequest(url); - if (style) { - rememberStyle(tabId, style); - const generation = scope.generation(tabId); - runForTab(tabId, async () => { - if (scope.generation(tabId) !== generation) return; - await applyStyleToStored(tabId, style); - }); - } - const service = classifyServiceRequest(url); - if (!service) return; - // A vector tileset is only addable with the source layers its style names. - if (service.format === "Vector tiles") { - service.styleUrl = stylesByTab.get(tabId)?.get(new URL(service.url).origin) ?? null; - } - // Identical to the entry already at the head of the list: nothing to write. - const key = candidateKey(service); - if (lastWritten.get(tabId) === key) return; - lastWritten.set(tabId, key); - const generation = scope.generation(tabId); - runForTab(tabId, async () => { - // The tab may have navigated while this write waited its turn. - if (scope.generation(tabId) !== generation) { - lastWritten.delete(tabId); - return; - } - const storageKey = `services:${tabId}`; - const stored = await chrome.storage.session.get(storageKey); - const existing = Array.isArray(stored[storageKey]) ? stored[storageKey] : []; - // One service can serve several layers, so an entry is a duplicate only - // when it repeats the layer too — and a repeat that has since picked up a - // style still replaces the entry that lacked one. - const same = (entry) => - Boolean(entry) && entry.url === service.url && (entry.layer ?? null) === service.layer; - const next = [service, ...existing.filter((entry) => !same(entry))].slice( - 0, - MAX_REQUESTS_PER_TAB, - ); - await chrome.storage.session.set({ [storageKey]: next }); - }); - }, - { urls: ["http://*/*", "https://*/*"] }, -); - -chrome.tabs.onRemoved.addListener((tabId) => { - scope.forget(tabId); - forgetTab(tabId); - runForTab(tabId, () => chrome.storage.session.remove(`services:${tabId}`)); -}); diff --git a/extensions/geolibre-chrome/manifest.json b/extensions/geolibre-chrome/manifest.json index 8f784f1d5..8b8261cf5 100644 --- a/extensions/geolibre-chrome/manifest.json +++ b/extensions/geolibre-chrome/manifest.json @@ -2,15 +2,10 @@ "manifest_version": 3, "name": "Open data in GeoLibre", "description": "Find geospatial datasets and services used by the current page and open them in GeoLibre.", - "version": "0.2.0", + "version": "0.3.0", "homepage_url": "https://geolibre.app/", "minimum_chrome_version": "106", - "permissions": ["activeTab", "scripting", "storage", "webRequest"], - "host_permissions": ["http://*/*", "https://*/*"], - "background": { - "service_worker": "background.mjs", - "type": "module" - }, + "permissions": ["activeTab", "scripting"], "action": { "default_title": "Open data in GeoLibre", "default_popup": "popup.html", diff --git a/extensions/geolibre-chrome/popup.mjs b/extensions/geolibre-chrome/popup.mjs index a54602dec..bf3784693 100644 --- a/extensions/geolibre-chrome/popup.mjs +++ b/extensions/geolibre-chrome/popup.mjs @@ -1,5 +1,5 @@ -import { scanDocumentForDatasets } from "./scanner.mjs"; -import { mergeServiceCandidates } from "./service-scanner.mjs"; +import { collectRequestedUrls, scanDocumentForDatasets } from "./scanner.mjs"; +import { collectServiceCandidates, mergeServiceCandidates } from "./service-scanner.mjs"; import { buildGeoLibreUrl } from "./url-builder.mjs"; const elements = { @@ -144,9 +144,19 @@ async function inspectPage() { } catch (error) { console.debug("GeoLibre could not scan the current document.", error); } - const key = `services:${tab.id}`; - const stored = await chrome.storage.session.get(key); - renderDatasets(mergeServiceCandidates(documentDatasets, stored[key] ?? [])); + let services = []; + try { + // A map can be embedded in a frame, and the requests are recorded by the + // document that made them, so every frame is asked for its own history. + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id, allFrames: true }, + func: collectRequestedUrls, + }); + services = collectServiceCandidates(results.flatMap((frame) => frame?.result ?? [])); + } catch (error) { + console.debug("GeoLibre could not read the page's requests.", error); + } + renderDatasets(mergeServiceCandidates(documentDatasets, services)); } elements.selectAll.addEventListener("click", () => { diff --git a/extensions/geolibre-chrome/scanner.mjs b/extensions/geolibre-chrome/scanner.mjs index e27479943..5399d1c20 100644 --- a/extensions/geolibre-chrome/scanner.mjs +++ b/extensions/geolibre-chrome/scanner.mjs @@ -292,3 +292,24 @@ export function scanDocumentForDatasets() { ) .map(({ confidence: _confidence, ...dataset }) => dataset); } + +/** + * Read back the URLs this document has already requested. A map fetches its + * tiles and service documents from JavaScript, so they are never links in the + * page and `scanDocumentForDatasets` cannot see them; the Resource Timing + * buffer is the record of them that the page keeps on its own behalf. + * + * Keep every helper inside this function: Chrome serializes it when it injects + * it into the active tab, so it cannot close over module-level values. + */ +export function collectRequestedUrls() { + try { + return performance + .getEntriesByType("resource") + .map((entry) => entry.name) + .filter((name) => name.startsWith("http:") || name.startsWith("https:")); + } catch { + // A document that denies the timing API leaves only its links to scan. + return []; + } +} diff --git a/extensions/geolibre-chrome/service-scanner.mjs b/extensions/geolibre-chrome/service-scanner.mjs index 328765d78..04d0b0a86 100644 --- a/extensions/geolibre-chrome/service-scanner.mjs +++ b/extensions/geolibre-chrome/service-scanner.mjs @@ -178,6 +178,15 @@ export function classifyServiceRequest(rawUrl) { return candidate(url, "OGC API", "vector", "OGC API service", api.href); } + // A MapLibre-style renderer fetches its `.pbf` tiles from a web worker, and a + // worker's requests are recorded in the worker's own timeline rather than the + // document's, so the TileJSON the main thread fetched to find them can be the + // only trace of the tileset. The body is never read, so a TileJSON describing + // raster tiles is indistinguishable here and is offered as a vector tileset. + if (/\/tile(?:s|json)\.json$/i.test(path)) { + return candidate(url, "Vector tiles", "vector", "Vector tile service"); + } + const tile = path.match(/^(.*\/)(\d+)\/(\d+)\/(\d+)(\.(?:png|jpe?g|webp|gif|pbf|mvt))(?:\/)?$/i); if (tile) { const [zoom, column, row] = tile.slice(2, 5).map(Number); @@ -247,108 +256,52 @@ export function mergeServiceCandidates(...groups) { return [...merged.values()]; } -/** Serialize asynchronous mutations independently for each browser tab. */ -export function createTabTaskQueue() { - const pending = new Map(); - return (tabId, task) => { - const previous = pending.get(tabId) ?? Promise.resolve(); - const next = previous.catch(() => undefined).then(task); - pending.set(tabId, next); - void next - .catch(() => undefined) - .finally(() => { - if (pending.get(tabId) === next) pending.delete(tabId); - }); - return next; - }; -} - -const MAX_TRACKED_DOCUMENTS = 200; - -function remember(documents, documentId) { - documents.add(documentId); - // A long-lived single-page app can churn through frames without ever - // navigating the tab, so keep the oldest ids from accumulating forever. - if (documents.size > MAX_TRACKED_DOCUMENTS) { - documents.delete(documents.values().next().value); - } -} +/** + * The Resource Timing buffer holds every request a document made, so a busy + * page can offer hundreds of tiles that collapse to a handful of services. + * This bounds what an unusually varied page can put in front of the user. + */ +const MAX_SERVICE_CANDIDATES = 100; /** - * Track which documents belong to a tab's *current* page, so a request left in - * flight by the page before it cannot be filed under the page after it. + * Turn the URLs a document has requested into the services GeoLibre can open. * - * Chrome does not put a `documentId` on a navigation request: a `main_frame` or - * `sub_frame` event describes a document that does not exist yet, and the id is - * only ever seen afterwards, on the requests that document itself makes. A - * page's documents therefore cannot be enumerated when it loads. What *can* be - * known is which documents belonged to the pages before it, so each navigation - * retires the ids seen so far and every later request is accepted unless its - * document was retired. A request with no id at all is a navigation of the tab - * being watched (a service URL opened directly) and belongs to the new page. + * Tiles and the style that describes them are separate requests and arrive in + * no fixed order, so the styles are collected first and paired afterwards: a + * vector tileset is only addable with the source layers its style names. */ -export function createPageScope() { - const tabs = new Map(); - - const stateFor = (tabId) => { - let state = tabs.get(tabId); - if (!state) { - state = { - generation: 0, - seen: new Set(), - leaving: new Set(), - retired: new Set(), - navigating: false, - }; - tabs.set(tabId, state); - } - return state; - }; - - return { - /** - * A navigation has started. Everything seen so far belongs to the page - * being left, so mark it for retirement now rather than when the navigation - * completes: a small tile or service request made by the *incoming* page - * can finish before its own HTML does, and retiring at completion would - * sweep up that new document along with the old ones. - */ - beginPage(tabId) { - const state = stateFor(tabId); - for (const documentId of state.seen) remember(state.leaving, documentId); - state.seen = new Set(); - state.navigating = true; - }, - /** Retire the outgoing page's documents and open a new generation. */ - startPage(tabId) { - const state = stateFor(tabId); - // Without an observed navigation start there is no separate set to - // retire, so fall back to retiring everything seen. - const outgoing = state.navigating ? state.leaving : state.seen; - for (const documentId of outgoing) remember(state.retired, documentId); - state.leaving = new Set(); - if (!state.navigating) state.seen = new Set(); - state.navigating = false; - state.generation += 1; - return state.generation; - }, - /** The current page's generation, for re-checking a queued write. */ - generation(tabId) { - return stateFor(tabId).generation; - }, - accepts(tabId, documentId) { - const state = stateFor(tabId); - if (!documentId) return true; - if (state.retired.has(documentId)) return false; - // A request from the outgoing page can still complete while its - // replacement loads. It belongs to the page on screen, so it is accepted, - // but its document stays marked for retirement: moving it back among the - // incoming page's documents would let it outlive the navigation. - if (!state.leaving.has(documentId)) remember(state.seen, documentId); - return true; - }, - forget(tabId) { - tabs.delete(tabId); - }, - }; +export function collectServiceCandidates(urls) { + const stylesByOrigin = new Map(); + const services = []; + for (const url of urls) { + const style = classifyStyleRequest(url); + if (style) stylesByOrigin.set(style.origin, style.url); + const service = classifyServiceRequest(url); + if (service) services.push(service); + } + for (const service of services) { + if (service.format !== "Vector tiles" || service.styleUrl) continue; + service.styleUrl = stylesByOrigin.get(new URL(service.url).origin) ?? null; + } + const merged = mergeServiceCandidates(services); + // Same reason as the TileJSON rule above: when a style names its tiles inline + // there is no metadata request either, and the worker's tile requests are + // invisible, so an origin that served a style but no tileset is offered + // through the style itself. GeoLibre resolves a vector layer from a style URL + // alone, reading the tile template and source layers out of the document. + for (const [origin, styleUrl] of stylesByOrigin) { + const covered = merged.some( + (entry) => entry.format === "Vector tiles" && new URL(entry.url).origin === origin, + ); + if (covered) continue; + merged.push({ + url: styleUrl, + name: "Vector tile style", + format: "Vector tiles", + kind: "vector", + styleUrl, + layer: null, + }); + } + return merged.slice(0, MAX_SERVICE_CANDIDATES); } diff --git a/extensions/geolibre-chrome/url-builder.mjs b/extensions/geolibre-chrome/url-builder.mjs index 5de220527..ce7ca5b31 100644 --- a/extensions/geolibre-chrome/url-builder.mjs +++ b/extensions/geolibre-chrome/url-builder.mjs @@ -30,7 +30,10 @@ export function buildGeoLibreUrl(datasets, baseUrl = GEOLIBRE_WEB_URL) { throw new Error("GeoLibre can only open HTTP or HTTPS service links."); } target.searchParams.set("add", serviceKinds[service.format]); - target.searchParams.set("serviceUrl", service.url); + // A tileset known only through its style has the style as its own URL, and + // GeoLibre reads the tiles out of that document: handing the same URL over + // as the service URL as well would have it parsed as TileJSON and fail. + if (service.url !== service.styleUrl) target.searchParams.set("serviceUrl", service.url); // The layer and style the page was rendering: without them the dialog opens // on an endpoint whose layer field is empty and cannot be submitted. if (service.layer) target.searchParams.set("serviceLayer", service.layer); diff --git a/scripts/package-chrome-extension.mjs b/scripts/package-chrome-extension.mjs index bb4f80bc4..66bf51872 100644 --- a/scripts/package-chrome-extension.mjs +++ b/scripts/package-chrome-extension.mjs @@ -11,7 +11,6 @@ const runtimeFiles = [ "popup.html", "popup.css", "popup.mjs", - "background.mjs", "scanner.mjs", "service-scanner.mjs", "url-builder.mjs", diff --git a/tests/chrome-extension.test.ts b/tests/chrome-extension.test.ts index 8e361b024..700d03c29 100644 --- a/tests/chrome-extension.test.ts +++ b/tests/chrome-extension.test.ts @@ -5,8 +5,7 @@ import { scanDocumentForDatasets } from "../extensions/geolibre-chrome/scanner.m import { classifyServiceRequest, classifyStyleRequest, - createPageScope, - createTabTaskQueue, + collectServiceCandidates, mergeServiceCandidates, } from "../extensions/geolibre-chrome/service-scanner.mjs"; import { buildGeoLibreUrl } from "../extensions/geolibre-chrome/url-builder.mjs"; @@ -549,179 +548,120 @@ describe("GeoLibre Chrome extension service request scanner", () => { assert.ok(first && second); assert.equal(first.url, second.url); }); +}); - it("serializes asynchronous work independently per tab", async () => { - const enqueue = createTabTaskQueue(); - const order: string[] = []; - let release!: () => void; - const blocked = new Promise((resolve) => { - release = resolve; - }); - const first = enqueue(7, async () => { - order.push("first:start"); - await blocked; - order.push("first:end"); - }); - const second = enqueue(7, async () => { - order.push("second"); - }); - const other = enqueue(8, async () => { - order.push("other"); - }); - await other; - assert.deepEqual(order, ["first:start", "other"]); - release(); - await Promise.all([first, second]); - assert.deepEqual(order, ["first:start", "other", "first:end", "second"]); - }); - - it("accepts the documents of the current page, including frames it opens later", () => { - const scope = createPageScope(); - scope.startPage(7); - // Chrome sends no documentId on the navigation itself; the ids arrive on - // the requests the page then makes, the top document and its frames alike. - assert.equal(scope.accepts(7, undefined), true); - assert.equal(scope.accepts(7, "top-document"), true); - assert.equal(scope.accepts(7, "child-document"), true); - }); - - it("keeps a document the incoming page created before its own navigation finished", () => { - const scope = createPageScope(); - scope.startPage(4); - scope.accepts(4, "old-document"); - // A tile the next page requests can complete before that page's HTML does, - // so the boundary is drawn when the navigation starts. - scope.beginPage(4); - assert.equal(scope.accepts(4, "new-document"), true); - scope.startPage(4); - assert.equal(scope.accepts(4, "new-document"), true); - assert.equal(scope.accepts(4, "old-document"), false); - }); - - it("retires an outgoing document even if it reports in mid-navigation", () => { - const scope = createPageScope(); - scope.startPage(6); - scope.accepts(6, "old-document"); - scope.beginPage(6); - // Still the page on screen, so the request counts, but the document must - // not escape retirement by reporting during the transition. - assert.equal(scope.accepts(6, "old-document"), true); - scope.startPage(6); - assert.equal(scope.accepts(6, "old-document"), false); - }); - - it("refuses a request left in flight by the page that was navigated away from", () => { - const scope = createPageScope(); - scope.startPage(3); - assert.equal(scope.accepts(3, "old-document"), true); - scope.startPage(3); - assert.equal(scope.accepts(3, "old-document"), false); - assert.equal(scope.accepts(3, "new-document"), true); - // A second navigation retires the page between, not only the first one. - scope.startPage(3); - assert.equal(scope.accepts(3, "new-document"), false); - }); - - it("scopes documents and generations to their own tab", () => { - const scope = createPageScope(); - scope.startPage(1); - scope.accepts(1, "shared-id"); - const generation = scope.generation(1); - scope.startPage(2); - assert.equal(scope.generation(1), generation); - assert.equal(scope.accepts(2, "shared-id"), true); - scope.startPage(1); - assert.notEqual(scope.generation(1), generation); - assert.equal(scope.accepts(1, "shared-id"), false); - assert.equal(scope.accepts(2, "shared-id"), true); - }); - - it("forgets a closed tab rather than growing a set per tab that ever existed", () => { - const scope = createPageScope(); - scope.startPage(9); - scope.accepts(9, "document"); - scope.startPage(9); - assert.equal(scope.accepts(9, "document"), false); - scope.forget(9); - assert.equal(scope.generation(9), 0); - assert.equal(scope.accepts(9, "document"), true); +describe("GeoLibre Chrome extension request history", () => { + it("pairs a vector tileset with the style requested after it", () => { + assert.deepEqual( + collectServiceCandidates([ + "https://tiles.example.com/roads/4/5/6.pbf", + "https://tiles.example.com/style.json", + ]).map((entry) => entry.styleUrl), + ["https://tiles.example.com/style.json"], + ); }); -}); -describe("GeoLibre Chrome extension request watcher", () => { - interface Details { - tabId?: number; - type?: string; - url: string; - documentId?: string; - } - type Listener = (details: Details) => void; - - // `background.mjs` registers its listeners against the extension APIs at - // import time, so the module is exercised through a stub of them. - async function loadWatcher() { - const store = new Map(); - const completed: Listener[] = []; - const navigations: Listener[] = []; - let writes = 0; - const addListener = (list: Listener[]) => (fn: Listener) => list.push(fn); - Object.assign(globalThis, { - chrome: { - webRequest: { - onCompleted: { addListener: addListener(completed) }, - onBeforeRequest: { addListener: addListener(navigations) }, - }, - tabs: { onRemoved: { addListener: () => undefined } }, - storage: { - session: { - get: async (key: string) => ({ [key]: store.get(key) }), - set: async (items: Record) => { - writes += 1; - for (const [name, value] of Object.entries(items)) store.set(name, value); - }, - remove: async (key: string) => { - store.delete(key); - }, - }, - }, - }, - }); - await import("../extensions/geolibre-chrome/background.mjs"); - const settle = async () => { - for (let turn = 0; turn < 8; turn += 1) await new Promise((r) => setTimeout(r, 0)); - }; - return { - services: () => (store.get("services:1") ?? []) as { url: string; styleUrl: string | null }[], - writes: () => writes, - async request(details: Details) { - const event = { tabId: 1, type: "xmlhttprequest", documentId: "doc", ...details }; - if (event.type === "main_frame") for (const fn of navigations) fn(event); - for (const fn of completed) fn(event); - await settle(); - }, - }; - } + it("leaves a tileset unpaired when the style belongs to another origin", () => { + // The unrelated style explains no tileset here, so it is offered as one of + // its own rather than attached to a tileset it does not describe. + assert.deepEqual( + collectServiceCandidates([ + "https://tiles.example.com/roads/4/5/6.pbf", + "https://other.example.com/style.json", + ]).map((entry) => [entry.url, entry.styleUrl]), + [ + ["https://tiles.example.com/roads/{z}/{x}/{y}.pbf", null], + ["https://other.example.com/style.json", "https://other.example.com/style.json"], + ], + ); + }); + + it("collapses the repeated tiles a panned map leaves behind", () => { + const found = collectServiceCandidates([ + "https://tiles.example.com/roads/4/5/6.pbf", + "https://tiles.example.com/roads/7/8/9.pbf", + "https://tiles.example.com/roads/1/2/3.pbf", + ]); + assert.equal(found.length, 1); + assert.equal(found[0].url, "https://tiles.example.com/roads/{z}/{x}/{y}.pbf"); + }); - it("fills in a style that arrives after the tiles it describes", async () => { - const watcher = await loadWatcher(); - await watcher.request({ type: "main_frame", url: "https://maps.example.com/", documentId: "" }); - await watcher.request({ url: "https://tiles.example.com/roads/4/5/6.pbf" }); + it("ignores the ordinary requests a page makes alongside its map", () => { assert.deepEqual( - watcher.services().map((entry) => entry.styleUrl), - [null], + collectServiceCandidates([ + "https://example.com/app.js", + "https://example.com/logo.png", + "https://example.com/api/users", + ]), + [], ); - // The style is a separate request and can finish after the first tile. - await watcher.request({ url: "https://tiles.example.com/style.json" }); + }); + + it("keeps two layers of one service apart", () => { assert.deepEqual( - watcher.services().map((entry) => entry.styleUrl), - ["https://tiles.example.com/style.json"], + collectServiceCandidates([ + "https://maps.example.com/wms?SERVICE=WMS&REQUEST=GetMap&LAYERS=roads", + "https://maps.example.com/wms?SERVICE=WMS&REQUEST=GetMap&LAYERS=rivers", + ]).map((entry) => entry.layer), + ["roads", "rivers"], ); + }); + + it("offers a worker-fetched tileset through the TileJSON the page fetched", () => { + // MapLibre fetches its tiles from a web worker, whose requests never reach + // the document's timing buffer, so the metadata request is the only trace. + const found = collectServiceCandidates([ + "https://demotiles.example.org/style.json", + "https://demotiles.example.org/tiles/tiles.json", + ]); + assert.deepEqual(found, [ + { + url: "https://demotiles.example.org/tiles/tiles.json", + name: "Vector tile service", + format: "Vector tiles", + kind: "vector", + styleUrl: "https://demotiles.example.org/style.json", + layer: null, + }, + ]); + }); - // Panning a map repeats one candidate; that must not keep rewriting it. - const before = watcher.writes(); - await watcher.request({ url: "https://tiles.example.com/roads/7/8/9.pbf" }); - await watcher.request({ url: "https://tiles.example.com/roads/1/2/3.pbf" }); - assert.equal(watcher.writes(), before); - assert.equal(watcher.services().length, 1); + it("falls back to the style when a page fetched no tileset metadata", () => { + const found = collectServiceCandidates(["https://tiles.example.com/style.json"]); + assert.deepEqual(found, [ + { + url: "https://tiles.example.com/style.json", + name: "Vector tile style", + format: "Vector tiles", + kind: "vector", + styleUrl: "https://tiles.example.com/style.json", + layer: null, + }, + ]); + // That candidate opens GeoLibre on the style alone, with no tileset URL to + // be parsed as TileJSON. + const link = new URL(buildGeoLibreUrl(found)); + assert.equal(link.searchParams.get("add"), "ogc-vector-tiles"); + assert.equal(link.searchParams.get("serviceUrl"), null); + assert.equal(link.searchParams.get("serviceStyle"), "https://tiles.example.com/style.json"); + }); + + it("does not repeat a style that already explains a tileset", () => { + assert.equal( + collectServiceCandidates([ + "https://tiles.example.com/style.json", + "https://tiles.example.com/roads/4/5/6.pbf", + ]).length, + 1, + ); + }); + + it("bounds what an unusually varied page can offer", () => { + const urls = Array.from( + { length: 150 }, + (_unused, index) => + `https://maps.example.com/wms?SERVICE=WMS&REQUEST=GetMap&LAYERS=l${index}`, + ); + assert.equal(collectServiceCandidates(urls).length, 100); }); }); diff --git a/tests/data-url.test.ts b/tests/data-url.test.ts index cd7c8dc7e..b70b71137 100644 --- a/tests/data-url.test.ts +++ b/tests/data-url.test.ts @@ -94,9 +94,31 @@ describe("serviceUrlParameter", () => { ); }); + it("opens a vector tileset carried by its style alone", () => { + // A style that names its tiles inline is the whole service: the tileset + // field stays empty and the style is resolved for both. + assert.deepEqual( + serviceUrlParameter( + "?add=ogc-vector-tiles&serviceStyle=https://tiles.example.com/style.json", + ), + { + kind: "ogc-vector-tiles", + url: "", + layer: null, + styleUrl: "https://tiles.example.com/style.json", + }, + ); + }); + it("rejects unsupported kinds and non-web URLs", () => { assert.equal(serviceUrlParameter("?add=bogus&serviceUrl=https://example.com"), null); assert.equal(serviceUrlParameter("?add=xyz&serviceUrl=file:///tmp/tiles"), null); + // Every other kind still needs a service URL: a style cannot stand in. + assert.equal( + serviceUrlParameter("?add=wms&serviceStyle=https://x.example.com/style.json"), + null, + ); + assert.equal(serviceUrlParameter("?add=ogc-vector-tiles"), null); }); }); From 6b7da56f83aa4aed44926dde1921c294ffcc973b Mon Sep 17 00:00:00 2001 From: giswqs Date: Tue, 18 Aug 2026 21:49:49 -0400 Subject: [PATCH 2/6] Address Claude review feedback - Assert at compile time that GeoLibreCogRenderEngine still matches the RenderEngine union it hand-mirrors from maplibre-gl-raster. types.ts is the public plugin API and must not hard-depend on that package's types, so the check lives next to the real import and fails typecheck on drift rather than letting a stale identifier reach control.setEngine(). - Record the mirror in CLAUDE.md alongside the others it documents. --- CLAUDE.md | 1 + packages/plugins/src/plugins/maplibre-raster.ts | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0918a89b3..67f6ffe25 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,6 +132,7 @@ The browser build proxies the sidecar at `/sidecar` (same-origin, no CORS); conf - `MAX_VECTOR_BYTES` (`packages/plugins/src/plugins/remote-file-formats.ts`) mirrors `MAX_REMOTE_FILE_BYTES`, an **internal, unexported** constant in `maplibre-gl-vector` (2 GiB — DuckDB-WASM holds remote file sizes in 32 bits). It cannot be imported, so whenever `maplibre-gl-vector` is bumped (in `packages/plugins/package.json`) — including Dependabot PRs — re-check `src/lib/utils/remote.ts` in that package and update the mirror if it moved. If it drifts, the remote-browse panels (Source Cooperative, Hugging Face) silently block GeoParquet the engine could now open, or offer an Add that is certain to fail. Updating the constant is enough: the limit the user is shown is rendered from it, not written into the copy. `remote-file-formats.ts` is the **single** home for this and the other format/reader/size rules those panels share — a per-panel copy would miss this check, so add new browse panels against that module rather than duplicating it (`source-coop-api.ts` re-exports it under its own names for compatibility). - `MAP_PANEL_SELECTOR` (`apps/geolibre-desktop/src/components/layout/RecordVideoDialog.tsx`) mirrors the **rendered** control class names from `maplibre-gl-components` — `maplibre-gl-html-control`, `maplibre-gl-legend`, `maplibre-gl-colorbar` — so the Record Video "Include map panels" option can rasterize those on-map overlays into the recording. These are the display elements, deliberately **not** the `*-gui-control` authoring editors. The classes are internal and unexported, so whenever `maplibre-gl-components` is bumped (in `packages/plugins/package.json`) — including Dependabot PRs — re-check them against the rendered controls and update the selector if they moved. If a class drifts, the option silently stops burning that panel into the video (or the checkbox never appears) with no build error. - `GLOBE_CONTROL_TOGGLE_SELECTOR` (`packages/map/src/globe-control-toggle.ts`) mirrors the class names MapLibre's own `GlobeControl` puts on its toggle button — `maplibregl-ctrl-globe` and `maplibregl-ctrl-globe-enabled`, swapped on every projection change. `MapCanvas` persists a projection change from a **click** on that button rather than from the `projectiontransition` event, because style initialization and project reconciliation emit that event too and a stale one overwrites the projection of a project that has just loaded. The classes are internal and unexported, so whenever `maplibre-gl` is bumped (including Dependabot PRs) run the frontend suite — `tests/globe-control-toggle.test.ts` builds a real `GlobeControl` and fails if the mirror stops matching. Without that check a renamed class silently stops persisting the user's projection, with no build error. +- `GeoLibreCogRenderEngine` (`packages/plugins/src/types.ts`) mirrors the `RenderEngine` union `maplibre-gl-raster` exports (`maplibre-gl-raster` | `cog-tiler-wasm` | `titiler`). It is hand-written rather than imported because `types.ts` is the public plugin-API surface and importing there would make that package's types a hard dependency of every external plugin. Unlike the mirrors above this one is checked by the **compiler**, not a test: `CogRenderEngineMirrorIsExact` in `packages/plugins/src/plugins/maplibre-raster.ts` asserts both directions of assignability against the real imported type, so a renamed or dropped engine identifier fails `npm run typecheck`. Nothing extra to do on a `maplibre-gl-raster` bump beyond letting the build run; without it a stale identifier would reach `control.setEngine()` as a string the control no longer recognizes, silently leaving the raster unrendered. - `propertySpecFor` (`packages/core/src/expressions.ts`) fabricates the **unexported** `StylePropertySpecification` shape that `@maplibre/maplibre-gl-style-spec`'s `createExpression` uses for expected-result-type enforcement (the Expression Builder's filter → boolean / color checks). The cast hides any contract change from the compiler, so whenever `@maplibre/maplibre-gl-style-spec` is bumped (including Dependabot PRs) run the frontend suite — the "enforces an expected result type" test in `tests/expressions.test.ts` fails if the shape stops being honored. - `DISTANCE_SEGMENTS` / `NON_DISTANCE_NAMES` (`apps/geolibre-desktop/src/lib/whitebox-distance-params.ts`) decide, by parameter *name*, which Whitebox parameters are ground distances and so get the Processing dialog's metric unit picker (GeoLibre#1540). The segments are generic (`tolerance`, `radius`, `length`, `resolution`), so a tool can carry a matching name that is not a length — `corridor_tolerance` is a 0-1 fraction. Those are safe today only because the picker is confined to tools whose every dataset input is a vector layer, and the colliding names happen to sit on imagery/LiDAR tools; that is a coincidence, not a guarantee. So whenever `geolibre-wasm` is bumped (in `packages/processing/package.json`) — including Dependabot PRs — scan the new catalog for a `double` matching the rule whose description reads as a fraction, ratio, angle or weight, and add it to `NON_DISTANCE_NAMES`. If one is missed, that tool's field offers metres and silently converts a dimensionless number as if it were a distance, with no build error. - UI strings are translatable via **react-i18next**; catalogs live in `apps/geolibre-desktop/src/i18n/locales/*.json` (`en.json` is the source of truth, typed by `i18next.d.ts`). Use `t()` for new user-facing strings; a `?locale`/`?lang` query param sets the embed language. The UI mirrors for right-to-left locales (Arabic), so style new components with Tailwind's logical utilities (`ms-`/`me-`/`ps-`/`pe-`/`text-start`/`border-s`/`start-`…), not the physical `ml-`/`left-` forms. See `docs/i18n.md`. diff --git a/packages/plugins/src/plugins/maplibre-raster.ts b/packages/plugins/src/plugins/maplibre-raster.ts index 2e988a137..0cfefca00 100644 --- a/packages/plugins/src/plugins/maplibre-raster.ts +++ b/packages/plugins/src/plugins/maplibre-raster.ts @@ -7,7 +7,7 @@ import type { RasterSampleDataset, RenderEngine, } from "maplibre-gl-raster"; -import type { GeoLibreAppAPI, GeoLibreMapControlPosition } from "../types"; +import type { GeoLibreAppAPI, GeoLibreCogRenderEngine, GeoLibreMapControlPosition } from "../types"; import { ensureMercatorProjection } from "./map-projection-utils"; import { ensureSharedDeckOverlay, @@ -450,6 +450,18 @@ export interface RasterVisualizationDefaults { */ export type RasterRenderEngine = RenderEngine; +// `GeoLibreCogRenderEngine` in ../types hand-mirrors this union: types.ts is the +// public plugin-API surface, so it must not make `maplibre-gl-raster`'s types a +// hard dependency of every external plugin. Nothing otherwise links the two, and +// a renamed or dropped identifier would reach `control.setEngine()` as a string +// the control no longer knows, with no build error. These assert both directions +// so a bump of `maplibre-gl-raster` fails `npm run typecheck` instead. +type Mirrors = never; +export type CogRenderEngineMirrorIsExact = [ + Mirrors, + Mirrors, +]; + /** * Applies a default RGB band triple once the header has loaded. * From 510520ff728055001efe05ab397827a529c5c0ac Mon Sep 17 00:00:00 2001 From: giswqs Date: Tue, 18 Aug 2026 21:57:38 -0400 Subject: [PATCH 3/6] Address review feedback - Reserve room under MAX_SERVICE_CANDIDATES for the style fallbacks: a page varied enough to fill the cap is mostly repeating layers of a few endpoints, while a fallback is the only trace its origin leaves at all. - Recognize the singular `tile.json` alongside `tiles.json` and `tilejson.json`, with a regression test over all three spellings. - Document in the README what the TileJSON sniff does not reach (a server that names its metadata otherwise, and a raster TileJSON, which cannot be told apart without reading a body this design cannot fetch). --- extensions/geolibre-chrome/README.md | 16 +++++++++--- .../geolibre-chrome/service-scanner.mjs | 12 ++++++--- tests/chrome-extension.test.ts | 25 +++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/extensions/geolibre-chrome/README.md b/extensions/geolibre-chrome/README.md index d5a4a4449..a6b61d425 100644 --- a/extensions/geolibre-chrome/README.md +++ b/extensions/geolibre-chrome/README.md @@ -64,9 +64,19 @@ Two consequences of reading the buffer rather than watching the network: - **Worker requests are invisible.** MapLibre and similar renderers fetch vector tiles from a web worker, which records them in the worker's own timeline, not the document's. Such a tileset is recovered from the metadata the main thread - *did* fetch: its TileJSON (`…/tiles.json`), or failing that the style document, - which GeoLibre can resolve a layer from on its own. A style is only offered in - its own right when no tileset from its origin was found. + *did* fetch: its TileJSON, or failing that the style document, which GeoLibre + can resolve a layer from on its own. A style is only offered in its own right + when no tileset from its origin was found. + + A TileJSON is recognized by name (`tile.json`, `tiles.json`, `tilejson.json`), + which is narrower than every way a server can name one: tileserver-gl serves + `/data/.json`, and that tileset is reached through its style rather than + its metadata. Nothing reads the response, so a TileJSON describing *raster* + tiles is indistinguishable from a vector one and is offered as a vector + tileset; selecting it opens the dialog on a document with no source layers, + which reports that plainly rather than drawing the wrong thing. Reading the + body would mean fetching a cross-origin URL, which needs the host permissions + this design exists to avoid. - **The buffer is finite.** It holds 250 entries per document by default and stops recording once full. A map's own early requests are normally well inside that, but a very busy page can lose a service added late. Raising the limit diff --git a/extensions/geolibre-chrome/service-scanner.mjs b/extensions/geolibre-chrome/service-scanner.mjs index 04d0b0a86..d25049ddd 100644 --- a/extensions/geolibre-chrome/service-scanner.mjs +++ b/extensions/geolibre-chrome/service-scanner.mjs @@ -183,7 +183,7 @@ export function classifyServiceRequest(rawUrl) { // document's, so the TileJSON the main thread fetched to find them can be the // only trace of the tileset. The body is never read, so a TileJSON describing // raster tiles is indistinguishable here and is offered as a vector tileset. - if (/\/tile(?:s|json)\.json$/i.test(path)) { + if (/\/tile(?:s?|json)\.json$/i.test(path)) { return candidate(url, "Vector tiles", "vector", "Vector tile service"); } @@ -289,12 +289,13 @@ export function collectServiceCandidates(urls) { // invisible, so an origin that served a style but no tileset is offered // through the style itself. GeoLibre resolves a vector layer from a style URL // alone, reading the tile template and source layers out of the document. + const fallbacks = []; for (const [origin, styleUrl] of stylesByOrigin) { const covered = merged.some( (entry) => entry.format === "Vector tiles" && new URL(entry.url).origin === origin, ); if (covered) continue; - merged.push({ + fallbacks.push({ url: styleUrl, name: "Vector tile style", format: "Vector tiles", @@ -303,5 +304,10 @@ export function collectServiceCandidates(urls) { layer: null, }); } - return merged.slice(0, MAX_SERVICE_CANDIDATES); + // The cap is taken out of the ordinary services first. A page varied enough to + // reach it is usually repeating layers of a few endpoints, while a fallback is + // the only trace its origin leaves at all, so spending every slot before + // reaching them would drop the one candidate that cannot be recovered. + const room = Math.max(0, MAX_SERVICE_CANDIDATES - fallbacks.length); + return [...merged.slice(0, room), ...fallbacks.slice(0, MAX_SERVICE_CANDIDATES)]; } diff --git a/tests/chrome-extension.test.ts b/tests/chrome-extension.test.ts index 700d03c29..72eea5d00 100644 --- a/tests/chrome-extension.test.ts +++ b/tests/chrome-extension.test.ts @@ -656,6 +656,31 @@ describe("GeoLibre Chrome extension request history", () => { ); }); + it("recognizes the singular and prefixed spellings of a TileJSON", () => { + for (const name of ["tile.json", "tiles.json", "tilejson.json"]) { + assert.deepEqual( + collectServiceCandidates([`https://tiles.example.com/data/${name}`]).map( + (entry) => entry.url, + ), + [`https://tiles.example.com/data/${name}`], + name, + ); + } + }); + + it("keeps room for a style fallback on a page that fills the cap", () => { + // A fallback is the only trace its origin leaves, while the services + // crowding it out are largely repeated layers of a few endpoints. + const urls = Array.from( + { length: 150 }, + (_unused, index) => + `https://maps.example.com/wms?SERVICE=WMS&REQUEST=GetMap&LAYERS=l${index}`, + ); + const found = collectServiceCandidates([...urls, "https://tiles.example.com/style.json"]); + assert.equal(found.length, 100); + assert.deepEqual(found.at(-1)?.url, "https://tiles.example.com/style.json"); + }); + it("bounds what an unusually varied page can offer", () => { const urls = Array.from( { length: 150 }, From 6555235494d5e6b67e368247a627a96b3b8f35e8 Mon Sep 17 00:00:00 2001 From: giswqs Date: Tue, 18 Aug 2026 22:37:31 -0400 Subject: [PATCH 4/6] Address review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Offer a style as a candidate of its own only when its path names it a map style (`…/style.json`, `…/styles.json`, an ArcGIS `…/resources/styles/ .json`). The looser `…/styles/.json` is an ordinary theme route too, so a style matched that way still explains a tileset at its origin but no longer surfaces on its own, where a page's theme file would appear as a layer. - State the raster-TileJSON failure precisely in the README: Add Data resolves the document on submit and refuses it when no source layers come out, rather than every selection opening on an empty dialog. --- extensions/geolibre-chrome/README.md | 15 ++++++++---- .../geolibre-chrome/service-scanner.mjs | 24 +++++++++++-------- tests/chrome-extension.test.ts | 23 ++++++++++++++++++ 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/extensions/geolibre-chrome/README.md b/extensions/geolibre-chrome/README.md index a6b61d425..2dd07cdc5 100644 --- a/extensions/geolibre-chrome/README.md +++ b/extensions/geolibre-chrome/README.md @@ -73,10 +73,17 @@ Two consequences of reading the buffer rather than watching the network: `/data/.json`, and that tileset is reached through its style rather than its metadata. Nothing reads the response, so a TileJSON describing *raster* tiles is indistinguishable from a vector one and is offered as a vector - tileset; selecting it opens the dialog on a document with no source layers, - which reports that plainly rather than drawing the wrong thing. Reading the - body would mean fetching a cross-origin URL, which needs the host permissions - this design exists to avoid. + tileset. Such a false positive cannot become a layer: Add Data resolves the + document on submit and refuses it when no source layers come out. Reading the + body here would mean fetching a cross-origin URL, which needs the host + permissions this design exists to avoid. + + A style stands as a candidate of its own only when its path names it one: + `…/style.json`, `…/styles.json`, or an ArcGIS `…/resources/styles/.json`. + The looser `…/styles/.json` is an ordinary theme or configuration route + as well, so a style matched that way is still trusted to explain a tileset + found at its origin, but never offered on its own, where a page's theme file + would appear as a layer. - **The buffer is finite.** It holds 250 entries per document by default and stops recording once full. A map's own early requests are normally well inside that, but a very busy page can lose a service added late. Raising the limit diff --git a/extensions/geolibre-chrome/service-scanner.mjs b/extensions/geolibre-chrome/service-scanner.mjs index d25049ddd..3955372b8 100644 --- a/extensions/geolibre-chrome/service-scanner.mjs +++ b/extensions/geolibre-chrome/service-scanner.mjs @@ -236,11 +236,14 @@ export function classifyStyleRequest(rawUrl) { } if (url.protocol !== "http:" && url.protocol !== "https:") return null; const path = url.pathname; - const isStyle = - /\/style(?:s)?\.json$/i.test(path) || - /\/styles?\/[^/]+\.json$/i.test(path) || - /\/resources\/styles\/[^/]*\.json$/i.test(path); - return isStyle ? { origin: url.origin, url: url.href } : null; + // `…/style.json` and an ArcGIS `…/resources/styles/.json` name themselves + // as map styles. `…/styles/.json` does not: a theme or configuration + // endpoint is served at exactly that path. So the generic spelling is trusted + // to explain a tileset already found at its origin, but never to become a + // candidate of its own, where it would offer a page's theme file as a layer. + const named = /\/styles?\.json$/i.test(path) || /\/resources\/styles\/[^/]*\.json$/i.test(path); + const isStyle = named || /\/styles?\/[^/]+\.json$/i.test(path); + return isStyle ? { origin: url.origin, url: url.href, named } : null; } /** Two entries describe the same thing only if they name the same layer. */ @@ -275,13 +278,13 @@ export function collectServiceCandidates(urls) { const services = []; for (const url of urls) { const style = classifyStyleRequest(url); - if (style) stylesByOrigin.set(style.origin, style.url); + if (style) stylesByOrigin.set(style.origin, style); const service = classifyServiceRequest(url); if (service) services.push(service); } for (const service of services) { if (service.format !== "Vector tiles" || service.styleUrl) continue; - service.styleUrl = stylesByOrigin.get(new URL(service.url).origin) ?? null; + service.styleUrl = stylesByOrigin.get(new URL(service.url).origin)?.url ?? null; } const merged = mergeServiceCandidates(services); // Same reason as the TileJSON rule above: when a style names its tiles inline @@ -290,17 +293,18 @@ export function collectServiceCandidates(urls) { // through the style itself. GeoLibre resolves a vector layer from a style URL // alone, reading the tile template and source layers out of the document. const fallbacks = []; - for (const [origin, styleUrl] of stylesByOrigin) { + for (const [origin, style] of stylesByOrigin) { + if (!style.named) continue; const covered = merged.some( (entry) => entry.format === "Vector tiles" && new URL(entry.url).origin === origin, ); if (covered) continue; fallbacks.push({ - url: styleUrl, + url: style.url, name: "Vector tile style", format: "Vector tiles", kind: "vector", - styleUrl, + styleUrl: style.url, layer: null, }); } diff --git a/tests/chrome-extension.test.ts b/tests/chrome-extension.test.ts index 72eea5d00..d93c7260f 100644 --- a/tests/chrome-extension.test.ts +++ b/tests/chrome-extension.test.ts @@ -444,6 +444,7 @@ describe("GeoLibre Chrome extension service request scanner", () => { assert.deepEqual(classifyStyleRequest("https://tiles.example.com/style.json"), { origin: "https://tiles.example.com", url: "https://tiles.example.com/style.json", + named: true, }); assert.equal( classifyStyleRequest("https://api.example.com/maps/streets/style.json?key=abc")?.url, @@ -646,6 +647,28 @@ describe("GeoLibre Chrome extension request history", () => { assert.equal(link.searchParams.get("serviceStyle"), "https://tiles.example.com/style.json"); }); + it("does not offer a theme file that merely sits at a style-shaped path", () => { + // `/styles/.json` is an ordinary theme and configuration route, so it + // explains a tileset found at its origin but never stands as one itself. + assert.deepEqual(collectServiceCandidates(["https://app.example.com/styles/dark.json"]), []); + assert.deepEqual( + collectServiceCandidates([ + "https://app.example.com/styles/dark.json", + "https://app.example.com/roads/4/5/6.pbf", + ]).map((entry) => entry.styleUrl), + ["https://app.example.com/styles/dark.json"], + ); + }); + + it("offers a style that names itself, including an ArcGIS one", () => { + assert.deepEqual( + collectServiceCandidates([ + "https://tiles.example.com/VectorTileServer/resources/styles/root.json", + ]).map((entry) => entry.url), + ["https://tiles.example.com/VectorTileServer/resources/styles/root.json"], + ); + }); + it("does not repeat a style that already explains a tileset", () => { assert.equal( collectServiceCandidates([ From 4d085f229a188f7978b6470c0ec8e7f97e4a5470 Mon Sep 17 00:00:00 2001 From: giswqs Date: Tue, 18 Aug 2026 23:03:21 -0400 Subject: [PATCH 5/6] Address review feedback - Keep a self-naming style over a generic one from the same origin. 65552354 made the fallback depend on that flag, but the map kept only the last style seen per origin, so a theme file fetched after `/style.json` stranded a worker-only tileset and handed an existing one the wrong style document. A test covers both request orders and fails without the fix. --- .../geolibre-chrome/service-scanner.mjs | 8 ++++++- tests/chrome-extension.test.ts | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/extensions/geolibre-chrome/service-scanner.mjs b/extensions/geolibre-chrome/service-scanner.mjs index 3955372b8..dbb85cc56 100644 --- a/extensions/geolibre-chrome/service-scanner.mjs +++ b/extensions/geolibre-chrome/service-scanner.mjs @@ -278,7 +278,13 @@ export function collectServiceCandidates(urls) { const services = []; for (const url of urls) { const style = classifyStyleRequest(url); - if (style) stylesByOrigin.set(style.origin, style); + // Keep a self-naming style over a generic one from the same origin. A page + // that fetches its map style and later a theme file at `…/styles/ + // .json` must still offer the map: letting the theme win would both strand + // a worker-only tileset and hand an existing one the wrong style document. + if (style && (!stylesByOrigin.get(style.origin)?.named || style.named)) { + stylesByOrigin.set(style.origin, style); + } const service = classifyServiceRequest(url); if (service) services.push(service); } diff --git a/tests/chrome-extension.test.ts b/tests/chrome-extension.test.ts index d93c7260f..8b5a4acef 100644 --- a/tests/chrome-extension.test.ts +++ b/tests/chrome-extension.test.ts @@ -660,6 +660,28 @@ describe("GeoLibre Chrome extension request history", () => { ); }); + it("keeps the map style when a theme file follows it from the same origin", () => { + // Either request order must leave the origin represented by its real style, + // both when the style is all there is and when it explains a tileset. + for (const order of [ + ["https://tiles.example.com/style.json", "https://tiles.example.com/styles/theme.json"], + ["https://tiles.example.com/styles/theme.json", "https://tiles.example.com/style.json"], + ]) { + assert.deepEqual( + collectServiceCandidates(order).map((entry) => entry.url), + ["https://tiles.example.com/style.json"], + order.join(" then "), + ); + assert.deepEqual( + collectServiceCandidates([...order, "https://tiles.example.com/roads/4/5/6.pbf"]).map( + (entry) => entry.styleUrl, + ), + ["https://tiles.example.com/style.json"], + order.join(" then "), + ); + } + }); + it("offers a style that names itself, including an ArcGIS one", () => { assert.deepEqual( collectServiceCandidates([ From 645fa0bd43157119051a360454a9b178afc250c0 Mon Sep 17 00:00:00 2001 From: Qiusheng Wu Date: Wed, 19 Aug 2026 13:04:42 -0400 Subject: [PATCH 6/6] Address review feedback - Document that `activeTab` does not reach cross-origin frames. Chrome grants the tab's main frame origin only and deliberately withholds that grant from a frame of another origin, so `allFrames: true` covers the top frame and its same-origin frames. The README described the scan as reading "each frame" and claimed the MapLibre row "covers services a page reaches only through an embedded frame", which over-claimed: that example's iframe is `src= "../display-a-map.html"`, same-origin with the docs page, so the live test never exercised a cross-origin frame. Added a third bullet to the buffer consequences, narrowed the two summary sentences and the table row, and recorded the boundary next to the `allFrames` call in popup.mjs. No code change: reaching such a frame needs standing host permission, which is exactly what this PR removes, so the limit is documented, not worked around. --- extensions/geolibre-chrome/README.md | 25 +++++++++++++++++++------ extensions/geolibre-chrome/popup.mjs | 6 ++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/extensions/geolibre-chrome/README.md b/extensions/geolibre-chrome/README.md index 2dd07cdc5..d2a360054 100644 --- a/extensions/geolibre-chrome/README.md +++ b/extensions/geolibre-chrome/README.md @@ -18,7 +18,8 @@ The published extension is on the Chrome Web Store: 4. Select this `extensions/geolibre-chrome` directory. Everything happens after you click the toolbar icon: the extension scans the -document's links, and reads each frame's Resource Timing buffer to recognize the +document's links, and reads each reachable frame's Resource Timing buffer to +recognize the services its maps requested. It holds no permission beyond `activeTab` and `scripting`, runs no background service worker, and stores nothing. @@ -54,12 +55,13 @@ type without changing the current selection. A map fetches its tiles and service documents from JavaScript, so they are never links in the page. What the extension reads instead is `performance .getEntriesByType("resource")`, the record of its own requests that every -document keeps, collected from the top frame and each frame below it. Recognized +document keeps, collected from the top frame and every same-origin frame below +it. Recognized are WMS, WMTS, WFS, OGC API Features, ArcGIS Feature Services, XYZ/TMS image tiles, and PBF/MVT vector tiles. Tile requests are collapsed into reusable `{z}/{x}/{y}` templates, and repeated requests from the same service appear once. -Two consequences of reading the buffer rather than watching the network: +Three consequences of reading the buffer rather than watching the network: - **Worker requests are invisible.** MapLibre and similar renderers fetch vector tiles from a web worker, which records them in the worker's own timeline, not @@ -89,6 +91,15 @@ Two consequences of reading the buffer rather than watching the network: that, but a very busy page can lose a service added late. Raising the limit needs a `document_start` script, which needs the broad host permissions this design exists to avoid, so the cap is accepted. +- **Cross-origin frames are out of reach.** `activeTab` grants the tab's main + frame origin, and Chrome deliberately does not extend that grant to a frame + from another origin. `allFrames: true` therefore reaches the top frame and its + same-origin frames; injection into a cross-origin frame is refused, and the + refusal is per-frame, so the frames that *are* reachable still return their + buffers. A map that runs entirely inside a cross-origin `iframe` is thus + invisible to the scan, and the popup reports no services rather than an error. + Reaching one needs host permission for that origin — the standing access this + design exists to avoid — so the boundary is accepted. A service endpoint on its own is rarely enough to add a layer, so each result also carries what the page asked that service *for*: the WMS `LAYERS` value, the @@ -116,14 +127,16 @@ map, so what it detects is what a real page actually requests. | OGC API Features | [pygeoapi lakes collection](https://demo.pygeoapi.io/master/collections/lakes/items?f=html) | `https://demo.pygeoapi.io/master/collections/lakes/items` | — | | ArcGIS Feature Service | [OpenLayers "Vector ESRI" example](https://openlayers.org/en/latest/examples/vector-esri.html) | The `…/FeatureServer/0` layer URL | `0` | | XYZ raster tiles | [openstreetmap.org](https://www.openstreetmap.org/) | `https://tile.openstreetmap.org/{z}/{x}/{y}.png` | — | -| Vector tiles, in an `iframe` | [MapLibre "Display a map" example](https://maplibre.org/maplibre-gl-js/docs/examples/display-a-map/) | `https://demotiles.maplibre.org/tiles/tiles.json` | style `…/style.json` | +| Vector tiles, in a same-origin `iframe` | [MapLibre "Display a map" example](https://maplibre.org/maplibre-gl-js/docs/examples/display-a-map/) | `https://demotiles.maplibre.org/tiles/tiles.json` | style `…/style.json` | Each row above adds a layer that draws, with no further typing: that is the bar for this table. A row that opens the dialog but leaves a required field empty is a bug, not an expected extra step. -The MapLibre row is worth keeping in the set: the map runs inside an `iframe`, so -it covers services a page reaches only through an embedded frame, and it renders +The MapLibre row is worth keeping in the set: the map runs inside a same-origin +`iframe` (the docs page embeds `../display-a-map.html`), so it covers services a +page reaches only through an embedded frame — the reachable kind, since a +cross-origin frame is outside `activeTab` — and it renders through a worker, so it covers the tileset recovered from its TileJSON rather than from a tile request. It also carries a style whose glyph ranges are served as `.pbf`; those are fonts, not a tileset, and must not be offered. diff --git a/extensions/geolibre-chrome/popup.mjs b/extensions/geolibre-chrome/popup.mjs index bf3784693..03b3b1365 100644 --- a/extensions/geolibre-chrome/popup.mjs +++ b/extensions/geolibre-chrome/popup.mjs @@ -148,6 +148,12 @@ async function inspectPage() { try { // A map can be embedded in a frame, and the requests are recorded by the // document that made them, so every frame is asked for its own history. + // `activeTab` grants only the tab's main frame origin, so Chrome refuses + // the injection into cross-origin frames — per frame, leaving the + // same-origin ones to still return their buffers. A map living wholly in a + // cross-origin frame is therefore not found; reaching it would need + // standing host permission, which this extension deliberately does not ask + // for. See the "Cross-origin frames are out of reach" note in README.md. const results = await chrome.scripting.executeScript({ target: { tabId: tab.id, allFrames: true }, func: collectRequestedUrls,