From e8f76d6bc2d1e17a1b80f51a4e48ba9f14043338 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 6 Jun 2026 21:47:03 +0200 Subject: [PATCH 1/4] Support EOPF multiscale HEALPix stores; drop EGI DataHub auth - Drive v3 indexing from the ROOT consolidated metadata for EOPF-style stores, where consolidated metadata lives only at the .zarr root and the level groups (under a multiscales group) have none. Find the multiscales group, enumerate the finest level's variables + cell_ids from the root tree, and build the index with store= + dataset=. - Resolve multiscales layout 'asset' paths against the .zarr root (they are root-relative), fixing the path-doubling that produced 404s and 'could not determine grid type'. - Remove EGI DataHub token auth: createFetchStore now returns a plain FetchStore; drop setAuthToken and the token URL param in HashGlobeView. Enables loading e.g. data.grid4earth.eu/sentinel-2-l2a/.zarr directly. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/data/authStore.ts | 23 ++------- src/lib/data/sourceIndexing.ts | 94 ++++++++++++++++++++++++++++++++-- src/views/HashGlobeView.vue | 8 --- 3 files changed, 94 insertions(+), 31 deletions(-) diff --git a/src/lib/data/authStore.ts b/src/lib/data/authStore.ts index 79894708..c26564af 100644 --- a/src/lib/data/authStore.ts +++ b/src/lib/data/authStore.ts @@ -1,27 +1,14 @@ import * as zarr from "zarrita"; -let authToken: string | null = null; - -export function setAuthToken(token: string | null) { - authToken = token; -} - -export function getAuthToken(): string | null { - return authToken; -} - /** - * Create a FetchStore with optional auth token injection. - * If an auth token is set, it is passed as an X-Auth-Token header - * via the FetchStore overrides mechanism. + * Create a FetchStore for public, CORS-enabled Zarr datasets. + * + * (EGI DataHub / token-authenticated store support has been removed in this + * build — datasets are expected to be publicly accessible.) */ export function createFetchStore( url: string | URL, options: { useSuffixRequest?: boolean } = {} ): zarr.FetchStore { - const overrides: RequestInit = {}; - if (authToken) { - overrides.headers = { "X-Auth-Token": authToken }; - } - return new zarr.FetchStore(url, { ...options, overrides }); + return new zarr.FetchStore(url, options); } diff --git a/src/lib/data/sourceIndexing.ts b/src/lib/data/sourceIndexing.ts index 2dbda582..690c8c58 100644 --- a/src/lib/data/sourceIndexing.ts +++ b/src/lib/data/sourceIndexing.ts @@ -191,6 +191,78 @@ function createIndex( }; } +/** The Zarr store root of a URL — everything up to and including ".zarr". */ +function zarrRoot(url: string): string { + const marker = ".zarr"; + const i = url.indexOf(marker); + return i >= 0 ? url.slice(0, i + marker.length) : url.replace(/\/+$/, ""); +} + +/** + * Build an index for an EOPF-style store, where consolidated metadata lives only + * at the Zarr root, a subgroup carries a `multiscales` layout (with root-relative + * `asset` paths), and each level group holds the variables + `cell_ids` but has no + * consolidated metadata of its own. We therefore drive everything from the root + * consolidated metadata instead of recursing into the (metadata-less) level groups. + */ +function buildEopfMultiscalesIndex( + rootSrc: string, + metadata: Record +): TSources | null { + let layout: Array<{ asset: string; [key: string]: unknown }> | undefined; + for (const node of Object.values(metadata)) { + const attrs = (node as { attributes?: Record }).attributes; + const ms = attrs?.multiscales as { layout?: typeof layout } | undefined; + if (node.node_type === "group" && ms?.layout?.length) { + layout = ms.layout; + break; + } + } + if (!layout?.length) { + return null; + } + + const finest = layout[0].asset.replace(/^\/+|\/+$/g, ""); // e.g. measurements/reflectance/20 + const prefix = finest + "/"; + const datasources: Record = {}; + for (const [key, node] of Object.entries(metadata)) { + if (node.node_type !== "array" || !key.startsWith(prefix)) { + continue; + } + const varname = key.slice(prefix.length); + if (varname.includes("/")) { + continue; // only direct children of the finest level group + } + const arrayNode = node as zarr.ArrayMetadata; + datasources[varname] = { + store: rootSrc, + dataset: finest, + hidden: + varname === "cell_ids" || + !isValidVariable(varname, arrayNode.shape, arrayNode.dimension_names), + attrs: { + ...node.attributes, + dimensionNames: arrayNode.dimension_names, + } as Record, + }; + } + if (Object.keys(datasources).length === 0) { + return null; + } + + return { + zarr_format: ZARR_FORMAT.V3, // eslint-disable-line camelcase + multiscales: { baseUrl: zarrRoot(rootSrc), layout }, + levels: [ + { + grid: { store: rootSrc, dataset: finest }, + time: { store: rootSrc, dataset: finest }, + datasources, + }, + ], + }; +} + export async function indexFromZarr(src: string): Promise { try { const store = await zarr.withConsolidated(lru(createFetchStore(src))); @@ -207,20 +279,32 @@ export async function indexFromZarr(src: string): Promise { createFetchStore(src) ); - // Detect multiscales metadata: redirect to the finest level subgroup // Note: openZarrV3Metadata puts the full zarr.json as group.attrs, // so actual group attributes are nested under group.attrs.attributes const rootAttrs = group.attrs as TZarrV3RootMetadata; const groupAttrs = (rootAttrs?.attributes ?? {}) as Record; + + // EOPF-style store: consolidated metadata only at the root, with a multiscales + // group whose level groups carry the variables. Drive the index from the root + // consolidated metadata (the level groups have no metadata to recurse into). + const consolidated = rootAttrs?.consolidated_metadata?.metadata; + if (consolidated) { + const eopf = buildEopfMultiscalesIndex(src.replace(/\/+$/, ""), consolidated); + if (eopf) { + return eopf; + } + } + + // multiscales attr directly on this group: redirect to the finest level, + // resolving the (root-relative) asset path against the Zarr root so the + // path is not doubled. if (groupAttrs?.multiscales) { const ms = groupAttrs.multiscales as { layout: Array<{ asset: string; [key: string]: unknown }>; }; if (ms.layout?.length > 0) { - const baseUrl = src.replace(/\/$/, ""); - const finestAsset = ms.layout[0].asset; - const levelUrl = baseUrl + "/" + finestAsset; - // Recursively index the subgroup (it's a regular zarr group) + const baseUrl = zarrRoot(src); + const levelUrl = baseUrl + "/" + ms.layout[0].asset.replace(/^\/+/, ""); const index = await indexFromZarr(levelUrl); index.multiscales = { baseUrl, layout: ms.layout }; return index; diff --git a/src/views/HashGlobeView.vue b/src/views/HashGlobeView.vue index 5a9314ee..1f37d674 100644 --- a/src/views/HashGlobeView.vue +++ b/src/views/HashGlobeView.vue @@ -5,7 +5,6 @@ import { ref, onBeforeMount, type Ref } from "vue"; import GlobeView from "./GlobeView.vue"; -import { setAuthToken } from "@/lib/data/authStore"; import { GRID_TYPES, type T_GRID_TYPES } from "@/lib/data/gridTypeDetector"; import { STORE_PARAM_MAPPING, useUrlParameterStore } from "@/store/paramStore"; import { useGlobeControlStore } from "@/store/store"; @@ -36,13 +35,6 @@ const onHashChange = () => { params.value = Object.fromEntries(new URLSearchParams(paramString)); - // Set auth token for authenticated Zarr stores (e.g. EGI DataHub) - if (params.value.token) { - setAuthToken(params.value.token as string); - } else { - setAuthToken(null); - } - if ( params.value.boundlow !== undefined && params.value.boundhigh !== undefined From 67ba40b37b22df35796f23963a6d573b051b0128 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 6 Jun 2026 22:21:14 +0200 Subject: [PATCH 2/4] Detect + size HEALPix from the dggs convention; fix per-level data path - gridTypeDetector: recognise HEALPix from the grid group's 'dggs' attribute (dggs.name == 'healpix'), not only a CF grid_mapping_name CRS variable. - Healpix.vue getHealpixCRSInfo: derive nside from dggs.refinement_level (nside = 2**level) and the ellipsoid from dggs.ellipsoid, falling back to the CF crs variable when present. - sourceIndexing (EOPF index): set datasource/grid/time store to the level-group URL with dataset="", so the multiscales LOD (rewriteDatasourcesUrl swaps the store per level) resolves variables at / instead of doubling the path. Verified: data.grid4earth.eu Sentinel-2 L2A EOPF store renders in the viewer. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/data/gridTypeDetector.ts | 14 +++++++++++ src/lib/data/sourceIndexing.ts | 14 +++++++---- src/ui/grids/Healpix.vue | 40 +++++++++++++++++++++----------- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/lib/data/gridTypeDetector.ts b/src/lib/data/gridTypeDetector.ts index e5193036..701a9d9e 100644 --- a/src/lib/data/gridTypeDetector.ts +++ b/src/lib/data/gridTypeDetector.ts @@ -162,6 +162,20 @@ export async function getGridType( varnameSelector ); + // DGGS convention (e.g. EOPF HEALPix): the grid group carries a `dggs` + // attribute instead of a CF `grid_mapping` CRS variable. + try { + const group = await ZarrDataManager.getDatasetGroup( + ZarrDataManager.getDatasetSource(datasources!, varnameSelector) + ); + const dggs = group.attrs?.dggs as { name?: string } | undefined; + if (typeof dggs?.name === "string" && dggs.name.toLowerCase() === "healpix") { + return GRID_TYPES.HEALPIX; + } + } catch { + // not a DGGS store — continue with the other checks + } + // Check CRS-based grid types const crsGridType = await determineGridTypeFromCRS( datasources!, diff --git a/src/lib/data/sourceIndexing.ts b/src/lib/data/sourceIndexing.ts index 690c8c58..5a0b1862 100644 --- a/src/lib/data/sourceIndexing.ts +++ b/src/lib/data/sourceIndexing.ts @@ -224,6 +224,10 @@ function buildEopfMultiscalesIndex( const finest = layout[0].asset.replace(/^\/+|\/+$/g, ""); // e.g. measurements/reflectance/20 const prefix = finest + "/"; + const root = zarrRoot(rootSrc); + // store points directly at the (finest) level group; the multiscales LOD swaps + // this store for other levels via rewriteDatasourcesUrl, so `dataset` stays "". + const finestUrl = root + "/" + finest; const datasources: Record = {}; for (const [key, node] of Object.entries(metadata)) { if (node.node_type !== "array" || !key.startsWith(prefix)) { @@ -235,8 +239,8 @@ function buildEopfMultiscalesIndex( } const arrayNode = node as zarr.ArrayMetadata; datasources[varname] = { - store: rootSrc, - dataset: finest, + store: finestUrl, + dataset: "", hidden: varname === "cell_ids" || !isValidVariable(varname, arrayNode.shape, arrayNode.dimension_names), @@ -252,11 +256,11 @@ function buildEopfMultiscalesIndex( return { zarr_format: ZARR_FORMAT.V3, // eslint-disable-line camelcase - multiscales: { baseUrl: zarrRoot(rootSrc), layout }, + multiscales: { baseUrl: root, layout }, levels: [ { - grid: { store: rootSrc, dataset: finest }, - time: { store: rootSrc, dataset: finest }, + grid: { store: finestUrl, dataset: "" }, + time: { store: finestUrl, dataset: "" }, datasources, }, ], diff --git a/src/ui/grids/Healpix.vue b/src/ui/grids/Healpix.vue index 946283fd..e0f93b7e 100644 --- a/src/ui/grids/Healpix.vue +++ b/src/ui/grids/Healpix.vue @@ -278,14 +278,11 @@ function fetchGrid() { } async function getHealpixCRSInfo() { - const crs = await ZarrDataManager.getCRSInfo( - activeDatasources.value!, - varnameSelector.value - ); - // FIXME: could probably have other names - const nside = crs.attrs["healpix_nside"] as number; - // Check DGGS convention for ellipsoid, then CRS attrs, then cell coordinate attrs + let nside: number | undefined; let ellipsoid: string | undefined; + + // DGGS convention first (e.g. EOPF): the grid group carries a `dggs` attribute + // with refinement_level (= HEALPix order, nside = 2**order) and an ellipsoid. try { const source = ZarrDataManager.getDatasetSource( activeDatasources.value!, @@ -293,15 +290,29 @@ async function getHealpixCRSInfo() { ); const group = await ZarrDataManager.getDatasetGroup(source); const dggs = group.attrs?.dggs as Record | undefined; - if (dggs?.ellipsoid) { - const ell = dggs.ellipsoid as Record; - ellipsoid = (ell.name as string)?.toUpperCase() || undefined; + if (dggs) { + if (typeof dggs.refinement_level === "number") { + nside = 2 ** (dggs.refinement_level as number); + } + if (dggs.ellipsoid) { + const ell = dggs.ellipsoid as Record; + ellipsoid = (ell.name as string)?.toUpperCase() || undefined; + } } } catch { - // Fall through + // Fall through to the CF crs-variable convention } - if (!ellipsoid) { - ellipsoid = (crs.attrs["healpix_ellipsoid"] as string) || undefined; + + // CF crs-variable convention (grid_mapping_name=healpix, healpix_nside) + if (nside === undefined) { + const crs = await ZarrDataManager.getCRSInfo( + activeDatasources.value!, + varnameSelector.value + ); + nside = crs.attrs["healpix_nside"] as number; + if (!ellipsoid) { + ellipsoid = (crs.attrs["healpix_ellipsoid"] as string) || undefined; + } } if (!ellipsoid) { try { @@ -325,6 +336,9 @@ async function getHealpixCRSInfo() { console.log("Could not read cell coordinate ellipsoid:", e); } } + if (nside === undefined) { + throw new Error("Could not determine HEALPix nside (no dggs or crs metadata)"); + } console.log("HEALPix CRS info: nside=", nside, "ellipsoid=", ellipsoid); return { nside, ellipsoid }; } From 65f236f6d143839f055fc66535c7009d948a67b6 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 6 Jun 2026 22:30:06 +0200 Subject: [PATCH 3/4] Satisfy lint: extract helpers + prettier formatting (zero eslint warnings) Extract checkDggsHealpix, findMultiscalesLayout, and readDggsCrsInfo so getGridType, buildEopfMultiscalesIndex and getHealpixCRSInfo stay under max-lines-per-function; apply prettier formatting. No behaviour change. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/data/gridTypeDetector.ts | 34 ++++++++++++++++--------- src/lib/data/sourceIndexing.ts | 27 ++++++++++++++------ src/ui/grids/Healpix.vue | 43 ++++++++++++++++++++------------ 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/src/lib/data/gridTypeDetector.ts b/src/lib/data/gridTypeDetector.ts index 701a9d9e..c4f230be 100644 --- a/src/lib/data/gridTypeDetector.ts +++ b/src/lib/data/gridTypeDetector.ts @@ -55,6 +55,25 @@ function checkHealpixGrid(crs: zarr.Array) { return crs.attrs["grid_mapping_name"] === "healpix"; } +// DGGS convention (e.g. EOPF HEALPix): the grid group carries a `dggs` attribute +// instead of a CF `grid_mapping` CRS variable. +async function checkDggsHealpix( + datasources: TSources, + varname: string +): Promise { + try { + const group = await ZarrDataManager.getDatasetGroup( + ZarrDataManager.getDatasetSource(datasources, varname) + ); + const dggs = group.attrs?.dggs as { name?: string } | undefined; + return ( + typeof dggs?.name === "string" && dggs.name.toLowerCase() === "healpix" + ); + } catch { + return false; + } +} + function checkRegularRotatedGrid( crs: zarr.Array ) { @@ -162,18 +181,9 @@ export async function getGridType( varnameSelector ); - // DGGS convention (e.g. EOPF HEALPix): the grid group carries a `dggs` - // attribute instead of a CF `grid_mapping` CRS variable. - try { - const group = await ZarrDataManager.getDatasetGroup( - ZarrDataManager.getDatasetSource(datasources!, varnameSelector) - ); - const dggs = group.attrs?.dggs as { name?: string } | undefined; - if (typeof dggs?.name === "string" && dggs.name.toLowerCase() === "healpix") { - return GRID_TYPES.HEALPIX; - } - } catch { - // not a DGGS store — continue with the other checks + // DGGS-convention HEALPix (e.g. EOPF) + if (await checkDggsHealpix(datasources!, varnameSelector)) { + return GRID_TYPES.HEALPIX; } // Check CRS-based grid types diff --git a/src/lib/data/sourceIndexing.ts b/src/lib/data/sourceIndexing.ts index 5a0b1862..ce52a436 100644 --- a/src/lib/data/sourceIndexing.ts +++ b/src/lib/data/sourceIndexing.ts @@ -205,19 +205,27 @@ function zarrRoot(url: string): string { * consolidated metadata of its own. We therefore drive everything from the root * consolidated metadata instead of recursing into the (metadata-less) level groups. */ -function buildEopfMultiscalesIndex( - rootSrc: string, +type TLayout = Array<{ asset: string; [key: string]: unknown }>; + +/** Find a `multiscales.layout` carried by any group node in the metadata tree. */ +function findMultiscalesLayout( metadata: Record -): TSources | null { - let layout: Array<{ asset: string; [key: string]: unknown }> | undefined; +): TLayout | undefined { for (const node of Object.values(metadata)) { const attrs = (node as { attributes?: Record }).attributes; - const ms = attrs?.multiscales as { layout?: typeof layout } | undefined; + const ms = attrs?.multiscales as { layout?: TLayout } | undefined; if (node.node_type === "group" && ms?.layout?.length) { - layout = ms.layout; - break; + return ms.layout; } } + return undefined; +} + +function buildEopfMultiscalesIndex( + rootSrc: string, + metadata: Record +): TSources | null { + const layout = findMultiscalesLayout(metadata); if (!layout?.length) { return null; } @@ -293,7 +301,10 @@ export async function indexFromZarr(src: string): Promise { // consolidated metadata (the level groups have no metadata to recurse into). const consolidated = rootAttrs?.consolidated_metadata?.metadata; if (consolidated) { - const eopf = buildEopfMultiscalesIndex(src.replace(/\/+$/, ""), consolidated); + const eopf = buildEopfMultiscalesIndex( + src.replace(/\/+$/, ""), + consolidated + ); if (eopf) { return eopf; } diff --git a/src/ui/grids/Healpix.vue b/src/ui/grids/Healpix.vue index e0f93b7e..7645d2a8 100644 --- a/src/ui/grids/Healpix.vue +++ b/src/ui/grids/Healpix.vue @@ -277,12 +277,11 @@ function fetchGrid() { } } -async function getHealpixCRSInfo() { - let nside: number | undefined; - let ellipsoid: string | undefined; - - // DGGS convention first (e.g. EOPF): the grid group carries a `dggs` attribute - // with refinement_level (= HEALPix order, nside = 2**order) and an ellipsoid. +// Read nside (from dggs.refinement_level) and ellipsoid from the group's `dggs`. +async function readDggsCrsInfo(): Promise<{ + nside?: number; + ellipsoid?: string; +}> { try { const source = ZarrDataManager.getDatasetSource( activeDatasources.value!, @@ -290,18 +289,28 @@ async function getHealpixCRSInfo() { ); const group = await ZarrDataManager.getDatasetGroup(source); const dggs = group.attrs?.dggs as Record | undefined; - if (dggs) { - if (typeof dggs.refinement_level === "number") { - nside = 2 ** (dggs.refinement_level as number); - } - if (dggs.ellipsoid) { - const ell = dggs.ellipsoid as Record; - ellipsoid = (ell.name as string)?.toUpperCase() || undefined; - } + if (!dggs) { + return {}; } + const nside = + typeof dggs.refinement_level === "number" + ? 2 ** dggs.refinement_level + : undefined; + const ell = dggs.ellipsoid as Record | undefined; + const ellipsoid = ell?.name + ? (ell.name as string).toUpperCase() + : undefined; + return { nside, ellipsoid }; } catch { - // Fall through to the CF crs-variable convention + return {}; } +} + +async function getHealpixCRSInfo() { + // DGGS convention first (e.g. EOPF): dggs.refinement_level -> nside. + const dggsInfo = await readDggsCrsInfo(); + let nside: number | undefined = dggsInfo.nside; + let ellipsoid: string | undefined = dggsInfo.ellipsoid; // CF crs-variable convention (grid_mapping_name=healpix, healpix_nside) if (nside === undefined) { @@ -337,7 +346,9 @@ async function getHealpixCRSInfo() { } } if (nside === undefined) { - throw new Error("Could not determine HEALPix nside (no dggs or crs metadata)"); + throw new Error( + "Could not determine HEALPix nside (no dggs or crs metadata)" + ); } console.log("HEALPix CRS info: nside=", nside, "ellipsoid=", ellipsoid); return { nside, ellipsoid }; From 44e5de9bba4c1a883d561927be4cf50890715354 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 6 Jun 2026 22:34:58 +0200 Subject: [PATCH 4/4] TEMP: deploy multiscale-eopf branch to Pages for preview Revert before/after merge (this branch should not stay in the deploy trigger). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 702f6b28..7e68225a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,7 +2,7 @@ name: Deploy to GitHub Pages on: push: - branches: [healpix-geo] + branches: [healpix-geo, multiscale-eopf] workflow_dispatch: permissions: