From 129c7643498e8a963384028f3a5ea14090acaf18 Mon Sep 17 00:00:00 2001 From: Martin Panknin Date: Thu, 1 Oct 2026 11:07:46 +0200 Subject: [PATCH] feat: open webKnossos datasets (x, y, z axes, URLs without .zarr) webKnossos publishes OME-Zarr layers as c, x, y, z with a transpose codec, under URLs without ".zarr". Kiln rejected the axis order and never tried Zarr for such URLs. - The spatial axes may come in any order within the last three dimensions; shape, chunks, scale, chunk coordinates and strides are read through that order. Nothing is transposed. - A URL is Kiln's sharded format only when it serves a volume.json; anything else is read as OME-Zarr. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 + docs/data/ome-zarr.md | 6 +- src/data/base-zarr-provider.ts | 53 ++++++++---------- src/data/chunk-math.ts | 34 ++++++++---- src/data/local-zarr-provider.ts | 6 +- src/data/sharded-provider.ts | 8 +++ src/data/zarr-chunk-worker.ts | 7 ++- src/data/zarr-validator.ts | 17 ++++-- src/engine.ts | 15 +++-- test/chunk-math.test.ts | 6 ++ test/webknossos-layout.test.ts | 97 +++++++++++++++++++++++++++++++++ test/zarr-validator.test.ts | 13 +++-- 12 files changed, 195 insertions(+), 70 deletions(-) create mode 100644 test/webknossos-layout.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5587ef59..e26d1001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] +### Added +- OME-Zarr with x, y, z axis order and Zarr URLs without `.zarr`, so public [webKnossos](https://webknossos.org) datasets open directly. + ## [0.5.0] - 2026-09-28 ### Added diff --git a/docs/data/ome-zarr.md b/docs/data/ome-zarr.md index 372ffe8c..3c9c7a25 100644 --- a/docs/data/ome-zarr.md +++ b/docs/data/ome-zarr.md @@ -15,7 +15,7 @@ Kiln is listed in the [OME-NGFF tools registry](https://ngff.openmicroscopy.org/ - **OME-NGFF v0.4 and v0.5** with `multiscales` metadata in group attributes - **Single-channel or multichannel** — up to 4 channels (see [Multichannel](/rendering/multichannel)) -- **3D arrays** with dimensions ordered `[z, y, x]` (standard C-order); multichannel datasets use a `c` axis +- **3D arrays** with x, y and z as the last three dimensions, usually `[z, y, x]`; `[x, y, z]` as written by [webKnossos](https://webknossos.org) works too. Multichannel datasets use a `c` axis - **Supported dtypes:** `uint8`, `uint16`, `int16`, `float32` (`float64` is read as `float32`). Other types, such as `int8` or `uint32`, are rejected with an explanation - Multiple resolution levels (datasets within `multiscales`) are used as LODs. Kiln reads each level's own size and per-axis downsampling (1× or 2×). If the stored levels don't fit that model, for example a coarse level a few slices short of half its parent, it falls back to a uniform 2:1 model and logs why. Force either model with `?pyramid=native` or `?pyramid=legacy` - Voxel spacing is read from `coordinateTransformations` if present @@ -33,7 +33,7 @@ import { KilnViewer } from 'kiln-render'; const viewer = await KilnViewer.create(canvas, 'https://example.com/data/scan.ome.zarr'); ``` -Kiln auto-detects the format from the URL. Brick assembly (fetching Zarr chunks, decompressing, and re-chunking into 66³ bricks with ghost borders) runs in a Web Worker pool off the main thread. For Zarr v2 stores (NGFF 0.4) the metadata is read in two round trips and shared with the workers, so they start without touching the network. +Any URL that doesn't serve Kiln's sharded format (a `volume.json`) is read as OME-Zarr, so URLs without `.zarr`, such as webKnossos layers, work as well. Brick assembly (fetching Zarr chunks, decompressing, and re-chunking into 66³ bricks with ghost borders) runs in a Web Worker pool off the main thread. For Zarr v2 stores (NGFF 0.4) the metadata is read in two round trips and shared with the workers, so they start without touching the network. ### Layout tips for fast streaming @@ -55,4 +55,4 @@ const viewer = await KilnViewer.create( ## Axis convention -Zarr stores dimensions as `[z, y, x]` (C-order, x fastest-varying). Kiln uses `[x, y, z]` in its metadata. Only metadata tuples are swapped; no data transposition is needed since the memory layout is identical. +Most OME-Zarr stores dimensions as `[z, y, x]`; webKnossos writes `[x, y, z]`. Kiln reads the order from the `axes` metadata and uses `[x, y, z]` internally. Chunks are read through their strides, so no data is transposed. diff --git a/src/data/base-zarr-provider.ts b/src/data/base-zarr-provider.ts index fe13cf25..e3a72d64 100644 --- a/src/data/base-zarr-provider.ts +++ b/src/data/base-zarr-provider.ts @@ -17,8 +17,9 @@ import type { import { valueTypeFor } from './value-types.js'; import { UnsupportedDatasetError } from './data-provider.js'; import { NetworkTracker } from './network-tracker.js'; -import { extractMultiscales, normalizeAxes, validateZarrSupport } from './zarr-validator.js'; -import { estimateBrickChunkFanout, computeBrickChunkFootprint } from './chunk-math.js'; +import { extractMultiscales, normalizeAxes, spatialOrder, validateZarrSupport } from './zarr-validator.js'; +import { estimateBrickChunkFanout, computeBrickChunkFootprint, pickXyz } from './chunk-math.js'; +import type { SpatialOrder } from './chunk-math.js'; import { buildPyramid } from '../core/pyramid.js'; import type { Vec3, PyramidPolicy } from '../core/pyramid.js'; @@ -53,12 +54,8 @@ export interface LodParams { channelAxisIdx: number; /** Channels stored per chunk along the channel axis (1 if none). */ channelChunkSize: number; -} - -/** Zarr vectors are [..., z, y, x]; Kiln works in [x, y, z]. */ -function lastThreeAsXyz(v: number[] | undefined): Vec3 | undefined { - if (!v || v.length < 3) return undefined; - return [v[v.length - 1]!, v[v.length - 2]!, v[v.length - 3]!]; + /** Positions of x, y, z within the last three dims; absent means the usual z, y, x. */ + spatialOrder?: SpatialOrder; } /** OMERO colours are "RRGGBB" hex strings; returns 0–1 RGB or undefined if malformed. */ @@ -257,6 +254,10 @@ export abstract class BaseZarrProvider implements DataProvider { const validationReasons = validateZarrSupport(ms, arrays[0]!.shape, String(dtype)); if (validationReasons.length > 0) throw new UnsupportedDatasetError(validationReasons); + // Kiln works in [x, y, z]; Zarr vectors end in z, y, x, or in x, y, z for webKnossos + const order = spatialOrder(axisNames, arrays[0]!.shape.length); + const xyz = (v: number[] | undefined): Vec3 | undefined => (v && v.length >= 3 ? pickXyz(v, order) : undefined); + // Storage mapping for the source dtype (bit depth, unorm vs raw-value path) const dtypeStr = String(dtype); const valueType = valueTypeFor(dtypeStr); @@ -268,34 +269,24 @@ export abstract class BaseZarrProvider implements DataProvider { let voxelSpacing: [number, number, number] | undefined; const transforms = ms.datasets[0]?.coordinateTransformations ?? ms.coordinateTransformations; if (transforms) { - const scaleTransform = transforms.find(t => t.type === 'scale'); - if (scaleTransform?.scale) { - const s = scaleTransform.scale; - // Zarr stores as [z, y, x], convert to [x, y, z] - voxelSpacing = [s[s.length - 1]!, s[s.length - 2]!, s[s.length - 3]!]; - } + voxelSpacing = xyz(transforms.find(t => t.type === 'scale')?.scale); } // Build LOD levels with virtual dimensions for uniform 2:1 downsampling - const lod0Shape = arrays[0]!.shape; // [z, y, x] - const lod0Dims: [number, number, number] = [ - lod0Shape[lod0Shape.length - 1]!, // x - lod0Shape[lod0Shape.length - 2]!, // y - lod0Shape[lod0Shape.length - 3]!, // z - ]; + const lod0Dims = pickXyz(arrays[0]!.shape, order); // Native pyramid from per-level transforms; validated but not yet driving geometry const pyramidBuild = buildPyramid( arrays.map((arr, i) => { const t = ms.datasets[i]?.coordinateTransformations ?? []; return { - dims: lastThreeAsXyz(arr.shape)!, - scale: lastThreeAsXyz(t.find(x => x.type === 'scale')?.scale), - translation: lastThreeAsXyz(t.find(x => x.type === 'translation')?.translation), + dims: pickXyz(arr.shape, order), + scale: xyz(t.find(x => x.type === 'scale')?.scale), + translation: xyz(t.find(x => x.type === 'translation')?.translation), }; }), - lastThreeAsXyz(ms.coordinateTransformations?.find(x => x.type === 'scale')?.scale), - lastThreeAsXyz(ms.coordinateTransformations?.find(x => x.type === 'translation')?.translation), + xyz(ms.coordinateTransformations?.find(x => x.type === 'scale')?.scale), + xyz(ms.coordinateTransformations?.find(x => x.type === 'translation')?.translation), ); const native = this.pyramidPolicy === 'native'; if (native && pyramidBuild.issues.length > 0) { @@ -308,9 +299,7 @@ export abstract class BaseZarrProvider implements DataProvider { const lodParams: LodParams[] = []; const levels: LodLevel[] = arrays.map((arr, i) => { const shape = arr.shape; - const actualDimX = shape[shape.length - 1]!; - const actualDimY = shape[shape.length - 2]!; - const actualDimZ = shape[shape.length - 3]!; + const [actualDimX, actualDimY, actualDimZ] = pickXyz(shape, order); // Native: levels are what the file stores. Legacy: uniform 2:1 virtual dims, resampled in assembly. const virtualDimX = native ? actualDimX : Math.ceil(lod0Dims[0] / (1 << i)); @@ -319,6 +308,7 @@ export abstract class BaseZarrProvider implements DataProvider { const chunkShape = arr.chunks; const shapePrefixLength = shape.length - 3; + const [csx, csy, csz] = pickXyz(chunkShape, order); lodParams.push({ scaleX: actualDimX / virtualDimX, scaleY: actualDimY / virtualDimY, @@ -326,12 +316,13 @@ export abstract class BaseZarrProvider implements DataProvider { actualDimX, actualDimY, actualDimZ, - csx: chunkShape[chunkShape.length - 1]!, - csy: chunkShape[chunkShape.length - 2]!, - csz: chunkShape[chunkShape.length - 3]!, + csx, + csy, + csz, shapePrefixLength, channelAxisIdx, channelChunkSize: channelAxisIdx >= 0 && channelAxisIdx < shapePrefixLength ? (chunkShape[channelAxisIdx] ?? 1) : 1, + spatialOrder: order, }); const brickGrid: [number, number, number] = [ diff --git a/src/data/chunk-math.ts b/src/data/chunk-math.ts index 84f96beb..5611efb4 100644 --- a/src/data/chunk-math.ts +++ b/src/data/chunk-math.ts @@ -93,16 +93,31 @@ export function estimateDatasetFanoutMultiplier(lodParamsList: LodChunkParams[], return Math.max(1, ...lodParamsList.map(p => estimateBrickChunkFanout(p, physSize))); } +/** Positions of x, y and z within the last three array dimensions. */ +export type SpatialOrder = [number, number, number]; + +/** The usual OME-Zarr layout, `…, z, y, x`. webKnossos writes `…, x, y, z`, which is [0, 1, 2]. */ +export const ZYX: SpatialOrder = [2, 1, 0]; + +/** The x, y and z entries of a per-dimension vector (shape, chunks, scale). */ +export function pickXyz(v: readonly number[], order: SpatialOrder = ZYX): [number, number, number] { + const b = v.length - 3; + return [v[b + order[0]]!, v[b + order[1]]!, v[b + order[2]]!]; +} + /** Zarr chunk coordinates for `channel`; non-spatial prefix axes other than channel are pinned to 0. */ export function chunkCoords( prefixLength: number, channelAxisIdx: number, channelChunkSize: number, - channel: number, cz: number, cy: number, cx: number, + channel: number, cz: number, cy: number, cx: number, order: SpatialOrder = ZYX, ): number[] { - const prefix = new Array(prefixLength).fill(0); + const coords = new Array(prefixLength + 3).fill(0); if (channelAxisIdx >= 0 && channelAxisIdx < prefixLength) { - prefix[channelAxisIdx] = Math.floor(channel / channelChunkSize); + coords[channelAxisIdx] = Math.floor(channel / channelChunkSize); } - return [...prefix, cz, cy, cx]; + coords[prefixLength + order[0]] = cx; + coords[prefixLength + order[1]] = cy; + coords[prefixLength + order[2]] = cz; + return coords; } /** Flat-index layout of a decoded chunk, valid for C, F and transposed storage orders. */ @@ -116,17 +131,12 @@ export interface ChunkLayout { export function chunkLayout( chunk: { shape: number[]; stride?: number[] }, - channelAxisIdx: number, channelChunkSize: number, channel: number, + channelAxisIdx: number, channelChunkSize: number, channel: number, order: SpatialOrder = ZYX, ): ChunkLayout { - const n = chunk.shape.length; const stride = chunk.stride ?? cOrderStrides(chunk.shape); const packedIdx = channelAxisIdx >= 0 && channelChunkSize > 1 ? channel % channelChunkSize : 0; - return { - base: packedIdx * (stride[channelAxisIdx] ?? 0), - strideX: stride[n - 1]!, - strideY: stride[n - 2]!, - strideZ: stride[n - 3]!, - }; + const [strideX, strideY, strideZ] = pickXyz(stride, order); + return { base: packedIdx * (stride[channelAxisIdx] ?? 0), strideX, strideY, strideZ }; } function cOrderStrides(shape: number[]): number[] { diff --git a/src/data/local-zarr-provider.ts b/src/data/local-zarr-provider.ts index 5f6e3c83..64c54dfa 100644 --- a/src/data/local-zarr-provider.ts +++ b/src/data/local-zarr-provider.ts @@ -134,7 +134,7 @@ export class LocalZarrDataProvider extends BaseZarrProvider { private async assembleBrick(lod: number, bx: number, by: number, bz: number, channelIndex = 0): Promise<{ data: BrickData; stats: BrickStats; rawMin?: number; rawMax?: number }> { const arr = this.arrays[lod]!; const params = this.lodParams[lod]!; - const { scaleX, scaleY, scaleZ, actualDimX, actualDimY, actualDimZ, csx, csy, csz, shapePrefixLength, channelAxisIdx, channelChunkSize } = params; + const { scaleX, scaleY, scaleZ, actualDimX, actualDimY, actualDimZ, csx, csy, csz, shapePrefixLength, channelAxisIdx, channelChunkSize, spatialOrder } = params; const physSize = this.metadata!.physicalBrickSize; const logicalSize = this.metadata!.brickSize; @@ -170,11 +170,11 @@ export class LocalZarrDataProvider extends BaseZarrProvider { for (let cy = minCy; cy <= maxCy; cy++) { for (let cx = minCx; cx <= maxCx; cx++) { const fi = (cz - minCz) * ncy * ncx + (cy - minCy) * ncx + (cx - minCx); - const coords = chunkCoords(shapePrefixLength, channelAxisIdx, channelChunkSize, channelIndex, cz, cy, cx); + const coords = chunkCoords(shapePrefixLength, channelAxisIdx, channelChunkSize, channelIndex, cz, cy, cx, spatialOrder); chunkFetches.push( arr.getChunk(coords).then(chunk => { chunkDataArr[fi] = chunk.data as unknown as ArrayLike; - const layout = chunkLayout(chunk, channelAxisIdx, channelChunkSize, channelIndex); + const layout = chunkLayout(chunk, channelAxisIdx, channelChunkSize, channelIndex, spatialOrder); chunkBase[fi] = layout.base; chunkStX[fi] = layout.strideX; chunkStY[fi] = layout.strideY; diff --git a/src/data/sharded-provider.ts b/src/data/sharded-provider.ts index 32c8a757..9b6366b7 100644 --- a/src/data/sharded-provider.ts +++ b/src/data/sharded-provider.ts @@ -65,6 +65,14 @@ interface ShardedBrickEntry { avg: number; } +/** A URL holds Kiln's sharded format when it serves a volume.json. Anything else is read + * as OME-Zarr, with or without ".zarr" in the URL (webKnossos layers have none). */ +export async function isShardedUrl(url: string): Promise { + if (url.includes('.zarr')) return false; + const response = await fetch(`${url.replace(/\/$/, '')}/volume.json`).catch(() => null); + return response?.ok === true; +} + /** * DataProvider implementation for Kiln's sharded binary format */ diff --git a/src/data/zarr-chunk-worker.ts b/src/data/zarr-chunk-worker.ts index 4d8a89e1..3086be9b 100644 --- a/src/data/zarr-chunk-worker.ts +++ b/src/data/zarr-chunk-worker.ts @@ -44,6 +44,7 @@ export interface ZarrWorkerRequest { shapePrefixLength: number; channelAxisIdx: number; channelChunkSize: number; + spatialOrder?: [number, number, number]; }[]; /** Channel index to load (for datasets with a channel axis) */ channelIndex?: number; @@ -342,7 +343,7 @@ async function assembleBrick( ): Promise<{ buffer: ArrayBuffer; min: number; max: number; avg: number; rawMin?: number; rawMax?: number; fetchMs: number; assemblyMs: number; chunkHits: number; chunkTotal: number }> { const arr = arrays[lod]!; const params = lodParams![lod]!; - const { scaleX, scaleY, scaleZ, actualDimX, actualDimY, actualDimZ, csx, csy, csz, shapePrefixLength, channelAxisIdx, channelChunkSize } = params; + const { scaleX, scaleY, scaleZ, actualDimX, actualDimY, actualDimZ, csx, csy, csz, shapePrefixLength, channelAxisIdx, channelChunkSize, spatialOrder } = params; const physSize = PHYSICAL_SIZE; const channelChunk = Math.floor(channelIndex / channelChunkSize); @@ -371,7 +372,7 @@ async function assembleBrick( const setChunkEntry = (fi: number, entry: DecodedChunk) => { chunkDataArr[fi] = entry.data; - const layout = chunkLayout(entry, channelAxisIdx, channelChunkSize, channelIndex); + const layout = chunkLayout(entry, channelAxisIdx, channelChunkSize, channelIndex, spatialOrder); chunkBase[fi] = layout.base; chunkStX[fi] = layout.strideX; chunkStY[fi] = layout.strideY; @@ -393,7 +394,7 @@ async function assembleBrick( setChunkEntry(fi, cached); chunkHits++; } else { - const coords = chunkCoords(shapePrefixLength, channelAxisIdx, channelChunkSize, channelIndex, cz, cy, cx); + const coords = chunkCoords(shapePrefixLength, channelAxisIdx, channelChunkSize, channelIndex, cz, cy, cx, spatialOrder); if (refcountedAborts) { // ?p4=1 — see fetchChunkShared. diff --git a/src/data/zarr-validator.ts b/src/data/zarr-validator.ts index 4479a96b..8ba0424e 100644 --- a/src/data/zarr-validator.ts +++ b/src/data/zarr-validator.ts @@ -7,6 +7,8 @@ import { open, root } from 'zarrita'; import { TolerantFetchStore } from './tolerant-fetch-store.js'; import { FileSystemStore } from './filesystem-store.js'; import { SUPPORTED_DTYPES } from './value-types.js'; +import { ZYX } from './chunk-math.js'; +import type { SpatialOrder } from './chunk-math.js'; interface MultiscalesEntry { axes?: unknown; // may be string[] (v0.4) or {name,type}[] (v0.5) or absent @@ -55,7 +57,7 @@ export function extractMultiscales(attrs: Record): MultiscalesE ); } -/** Assembly indexes the last three array dims as [z, y, x]; reject layouts that do not match. */ +/** Assembly reads the volume from the last three array dims; reject layouts that do not match. */ function validateSpatialLayout(axes: NormalizedAxis[], axesProvided: boolean, rank: number): string[] { if (rank < 3) return [`Array has ${rank} dimensions — a volume needs at least 3`]; if (!axesProvided) return []; @@ -64,14 +66,17 @@ function validateSpatialLayout(axes: NormalizedAxis[], axesProvided: boolean, ra const spatial = axes.map((a, i) => ({ ...a, i })).filter(a => a.type === 'space'); if (spatial.length !== 3) return [`Expected 3 spatial axes, found ${spatial.length}`]; if (spatial.some((a, k) => a.i !== rank - 3 + k)) return ['Spatial axes must be the last three array dimensions (…, z, y, x)']; - - const names = spatial.map(a => a.name.toLowerCase()); - if (names.every(n => 'xyz'.includes(n)) && names.join('') !== 'zyx') { - return [`Spatial axis order "${names.join(', ')}" is not supported (expected z, y, x)`]; - } return []; } +/** Where x, y and z sit within the last three array dimensions, from the axis names. + * Unnamed or unrecognised spatial axes are taken as the usual z, y, x. */ +export function spatialOrder(axes: NormalizedAxis[], rank: number): SpatialOrder { + const names = axes.length === rank ? axes.slice(-3).map(a => a.name.toLowerCase()) : []; + const order = ['x', 'y', 'z'].map(n => names.indexOf(n)); + return order.every(i => i >= 0) ? order as SpatialOrder : ZYX; +} + /** * Validate whether a dataset is supported. * Returns a list of human-readable reasons; empty array means fully supported. diff --git a/src/engine.ts b/src/engine.ts index 632f8820..03205da5 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -18,7 +18,7 @@ import type { ViewParams } from './core/view.js'; import type { DataProvider, VolumeMetadata } from './data/data-provider.js'; import { UnsupportedDatasetError } from './data/data-provider.js'; import { MAX_LOD_LEVELS } from './shaders/uniform-layout.js'; -import { ShardedDataProvider } from './data/sharded-provider.js'; +import { ShardedDataProvider, isShardedUrl } from './data/sharded-provider.js'; import { ZarrDataProvider } from './data/zarr-provider.js'; import { initializeWithPyramidFallback } from './data/pyramid-fallback.js'; @@ -195,10 +195,10 @@ export class KilnEngine { if (isExternalProvider) { dataProvider = dataset as DataProvider; } else { - const isZarr = (dataset as string).includes('.zarr'); - dataProvider = isZarr - ? new ZarrDataProvider(dataset as string) - : new ShardedDataProvider(dataset as string); + const url = dataset as string; + dataProvider = (await isShardedUrl(url)) + ? new ShardedDataProvider(url) + : new ZarrDataProvider(url); } // The level model must reach the provider before workers and brick loads exist. @@ -231,9 +231,8 @@ export class KilnEngine { // Configure worker target format (string-URL providers only) if (!isExternalProvider) { - const isHttpZarr = (dataset as string).includes('.zarr'); - if (isHttpZarr) { - await (dataProvider as ZarrDataProvider).setTargetFormat( + if (dataProvider instanceof ZarrDataProvider) { + await dataProvider.setTargetFormat( textureFormat as 'r8unorm' | 'r16float', ); } else if (textureFormat !== 'r16unorm' || sourceBitDepth !== 16) { diff --git a/test/chunk-math.test.ts b/test/chunk-math.test.ts index f8a8cdee..3ec3c68b 100644 --- a/test/chunk-math.test.ts +++ b/test/chunk-math.test.ts @@ -279,4 +279,10 @@ describe('chunkCoords / chunkLayout (storage order and packed channels)', () => it('unpacked channel chunks never add a base offset', () => { expect(chunkLayout({ shape: [1, 4, 8, 16] }, 0, 1, 3).base).toBe(0); }); + + it('x, y, z order (webKnossos): coordinates and strides follow the axes', () => { + expect(chunkCoords(1, 0, 1, 0, 4, 5, 6, [0, 1, 2])).toEqual([0, 6, 5, 4]); + const l = chunkLayout({ shape: [1, 16, 8, 4], stride: [1, 1, 16, 128] }, 0, 1, 0, [0, 1, 2]); + expect(l).toEqual({ base: 0, strideX: 1, strideY: 16, strideZ: 128 }); + }); }); diff --git a/test/webknossos-layout.test.ts b/test/webknossos-layout.test.ts new file mode 100644 index 00000000..f9e403ce --- /dev/null +++ b/test/webknossos-layout.test.ts @@ -0,0 +1,97 @@ +/** + * webKnossos publishes OME-Zarr layers as c, x, y, z with a transpose codec, under + * URLs without ".zarr". Metadata from the public Neuromast_WT1 layer on data-humerus. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { open, root } from 'zarrita'; +import { LocalZarrDataProvider } from '../src/data/local-zarr-provider.js'; +import { isShardedUrl } from '../src/data/sharded-provider.js'; + +const axes = [ + { name: 'c', type: 'channel' }, + { name: 'x', type: 'space', unit: 'nanometer' }, + { name: 'y', type: 'space', unit: 'nanometer' }, + { name: 'z', type: 'space', unit: 'nanometer' }, +]; +const attrs = (mags: number[]) => ({ + ome: { + version: '0.5', + multiscales: [{ + name: 'color', axes, + datasets: mags.map(m => ({ + path: String(m), coordinateTransformations: [{ type: 'scale', scale: [1, 6 * m, 6 * m, 30 * m] }], + })), + }], + }, +}); + +describe('webKnossos layout: metadata', () => { + const p: any = new LocalZarrDataProvider({ name: 'wk' } as any); + const level = (m: number) => ({ shape: [1, 9216 / m, 9216 / m, 1024 / m], chunks: [1, 32, 32, 32], dtype: 'uint8' }); + const { metadata, lodParams } = p.parseOmeMetadata(attrs([1, 2, 4]), [level(1), level(2), level(4)], 'wk'); + + it('reads dimensions, spacing and chunks as x, y, z', () => { + expect(metadata.dimensions).toEqual([9216, 9216, 1024]); + expect(metadata.voxelSpacing).toEqual([6, 6, 30]); + expect(lodParams[0]).toMatchObject({ csx: 32, csy: 32, csz: 32, actualDimZ: 1024, spatialOrder: [0, 1, 2] }); + }); + + it('builds the native pyramid', () => { + expect(metadata.pyramidIssues).toEqual([]); + expect(metadata.levels[2].dimensions).toEqual([2304, 2304, 256]); + }); +}); + +describe('webKnossos layout: voxels', () => { + it('every voxel lands at its x, y, z position through real decoding', async () => { + const X = 4, Y = 3, Z = 2; + const value = (x: number, y: number, z: number) => z * 100 + y * 10 + x; + const store = new Map(); + store.set('/zarr.json', new TextEncoder().encode(JSON.stringify({ + zarr_format: 3, node_type: 'array', shape: [1, X, Y, Z], data_type: 'uint8', + chunk_grid: { name: 'regular', configuration: { chunk_shape: [1, X, Y, Z] } }, + chunk_key_encoding: { name: 'v2', configuration: { separator: '.' } }, + fill_value: 0, attributes: {}, dimension_names: ['c', 'x', 'y', 'z'], + codecs: [{ name: 'transpose', configuration: { order: [3, 2, 1, 0] } }, { name: 'bytes', configuration: { endian: 'little' } }], + }))); + const data = new Uint8Array(X * Y * Z); // transposed to z, y, x, c: x varies fastest + for (let z = 0; z < Z; z++) for (let y = 0; y < Y; y++) for (let x = 0; x < X; x++) data[(z * Y + y) * X + x] = value(x, y, z); + store.set('/0.0.0.0', data); + + const arr = await open(root(store), { kind: 'array' }); + const p: any = new LocalZarrDataProvider({ name: 'wk' } as any); + const { metadata, lodParams } = p.parseOmeMetadata(attrs([1]), [arr], 'wk'); + Object.assign(p, { metadata, lodParams, arrays: [arr] }); + + const brick = (await p.loadBrick(0, 0, 0, 0)).data as Uint8Array; + const core = (x: number, y: number, z: number) => brick[(z + 1) * 66 * 66 + (y + 1) * 66 + (x + 1)]; + for (let z = 0; z < Z; z++) for (let y = 0; y < Y; y++) for (let x = 0; x < X; x++) { + expect(core(x, y, z)).toBe(value(x, y, z)); + } + }); +}); + +describe('isShardedUrl', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('takes a .zarr URL as Zarr without a request', async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal('fetch', fetchSpy); + expect(await isShardedUrl('https://example.org/scan.ome.zarr')).toBe(false); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('takes a URL that serves volume.json as sharded', async () => { + const fetchSpy = vi.fn().mockResolvedValue({ ok: true }); + vi.stubGlobal('fetch', fetchSpy); + expect(await isShardedUrl('https://example.org/chameleon/')).toBe(true); + expect(fetchSpy).toHaveBeenCalledWith('https://example.org/chameleon/volume.json'); + }); + + it('takes any other URL as Zarr, such as a webKnossos layer', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false })); + expect(await isShardedUrl('https://data-humerus.webknossos.org/data/zarr3_experimental/abc/color')).toBe(false); + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new TypeError('Failed to fetch'))); + expect(await isShardedUrl('https://data-humerus.webknossos.org/data/zarr3_experimental/abc/color')).toBe(false); + }); +}); diff --git a/test/zarr-validator.test.ts b/test/zarr-validator.test.ts index 35eb3d0f..2c9fa0be 100644 --- a/test/zarr-validator.test.ts +++ b/test/zarr-validator.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; -import { normalizeAxes, extractMultiscales, validateZarrSupport } from '../src/data/zarr-validator.js'; +import { normalizeAxes, extractMultiscales, validateZarrSupport, spatialOrder } from '../src/data/zarr-validator.js'; // --------------------------------------------------------------------------- // normalizeAxes @@ -260,9 +260,14 @@ describe('validateZarrSupport — spatial layout', () => { expect(reasons.join()).toMatch(/3 spatial axes, found 2/); }); - it('rejects x, y, z order', () => { - const reasons = validateZarrSupport(ms(['x', 'y', 'z']), [64, 64, 64], 'uint8'); - expect(reasons.join()).toMatch(/expected z, y, x/); + it('accepts x, y, z order (webKnossos)', () => { + expect(validateZarrSupport(ms(['c', 'x', 'y', 'z']), [1, 64, 64, 64], 'uint8')).toEqual([]); + }); + + it('finds x, y and z within the last three dimensions', () => { + expect(spatialOrder(normalizeAxes(['c', 'x', 'y', 'z']), 4)).toEqual([0, 1, 2]); + expect(spatialOrder(normalizeAxes(['t', 'c', 'z', 'y', 'x']), 5)).toEqual([2, 1, 0]); + expect(spatialOrder(normalizeAxes(undefined), 4)).toEqual([2, 1, 0]); // no axes: z, y, x }); it('rejects a spatial axis in the prefix', () => {