From 1dad34bab0e55a0b512c78e529cca6de0c16b38e Mon Sep 17 00:00:00 2001 From: aaronkanzer <36093535+aaronkanzer@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:38:15 -0400 Subject: [PATCH 1/2] edge: draft BIL sheets from OME-Zarr stores and voxel manifests Two new scigantic-edge record formats reach submission_draft(s): - ome-zarr: step sizes from the level-0 scale in the axes' unit (the _um table now reads NGFF unit names, micro sign, Greek mu, plurals and British spellings), channels/slices/size from the shape, dimensionorder from the axes names (CZYX, TCZYX). - voxel-manifest: Instrument sheet from the manifest of an Allen Institute voxel acquisition (instrument name, objective, laser lines, emission bands, camera). Light-sheet is claimed only when the instrument's name says so; the template instrument is a simulator. A voxel acquisition is one store per volume and channel under /volumes/NNNNNN//.ome.zarr. Stores collapse into one draft per manifest: by the store's manifest sidecar, by the manifest's storePaths, or by the /volumes/ prefix when no manifest is indexed yet; otherwise a store groups under its parent directory like any other file. channels is the count of distinct channel ids across the volumes, not the sizeC of one store. A new draft key, acquisition, carries store and volume counts, channel names and the manifest's status, revision and timestamps. fixtures/edge/voxel_run1_index.jsonl is the edge decoder's index over a real headless run of voxel's simulated instrument; the exaSPIM fixture is hand-written from two public tiles' metadata. The prior is unchanged: its bil-image fields already line up with what an ome-zarr record says. --- README.md | 2 + fixtures/edge/exaspim_public_index.jsonl | 2 + fixtures/edge/voxel_run1_index.jsonl | 5 + src/scigantic_bil/edge.py | 246 +++++++++++++++++++++-- tests/test_edge.py | 146 ++++++++++++++ 5 files changed, 386 insertions(+), 15 deletions(-) create mode 100644 fixtures/edge/exaspim_public_index.jsonl create mode 100644 fixtures/edge/voxel_run1_index.jsonl diff --git a/README.md b/README.md index 173b385..4717089 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,8 @@ $ scigantic-bil submission-draft ./ctx # one draft per acquisition di The prior is the same base64 capsule the edge's own `-export-prior` writes; its card holds one `bil-image` context record per Image row of the BIL record, with the Instrument sheet folded in. The draft uses BIL's own column names, so a row pastes into the submission spreadsheet. Measured on 2026-10-17: of 1,004 light-sheet records on BIL, 849 have an Instrument block and about half of those name an objective, detector or wavelength; 808 are LifeCanvas SmartSPIM, whose `metadata.txt` carries every one of those values. +OME-Zarr stores are drafted from the edge's `ome-zarr` records: step sizes from the level-0 scale transform (converted to micrometres from whatever unit the axes name), channels, slices and array size from the shape, `dimensionorder` from the axes names (`CZYX`, `TCZYX`). An acquisition written by the Allen Institute's `voxel` engine is one store per volume and channel under `/volumes/NNNNNN//.ome.zarr` beside a `manifest.json`; those stores collapse into one draft per manifest (matched through the store's sidecar, the manifest's store paths, or the `/volumes/` prefix), with `channels` counted as the distinct `.ome.zarr` names across volumes. The manifest's `voxel-manifest` record fills the Instrument sheet (instrument name, objective, laser lines, emission bands, camera) and the draft's `acquisition` block carries what the sheets have no column for: store and volume counts, channel names, status, revision and timestamps. The microscope type is set to light-sheet only when the instrument's name says so; voxel's template instrument is a simulator. `fixtures/edge/voxel_run1_index.jsonl` is the edge decoder's own `index.jsonl` over one such run (2 tiles x 2 channels, 10 frames of 128 x 192 uint16), `exaspim_public_index.jsonl` holds hand-written records for two public exaSPIM tiles, and `tests/test_edge.py` reads both. + ## Caching On by default. Metadata responses, directory listings and manifests are cached to `~/.cache/scigantic-bil` (macOS: `~/Library/Caches/scigantic-bil`; override with `SCIGANTIC_BIL_CACHE` or `enable_cache(cache_dir=...)`) and expire after 7 days, since BIL republishes its inventory every few days. Inventory snapshots are immutable once published and are kept as plain TSV files without expiry. Image bytes are never cached. diff --git a/fixtures/edge/exaspim_public_index.jsonl b/fixtures/edge/exaspim_public_index.jsonl new file mode 100644 index 0000000..b13c684 --- /dev/null +++ b/fixtures/edge/exaspim_public_index.jsonl @@ -0,0 +1,2 @@ +{"source": "exaSPIM_851474_2026-07-10_15-23-22/SPIM.ome.zarr/tile_000000_ch_488.zarr", "sizeBytes": 7759, "mtime": 1784108793, "decoded": {"format": "ome-zarr", "summary": "OME-Zarr v2 (NGFF 0.4) multiscales, 7 levels, tczyx 1x1x2560x1330x1774 uint16, 8 x 5.984 x 5.984 micrometer, blosc/zstd", "fields": {"zarrFormat": 2, "ngffVersion": "0.4", "layout": "multiscales", "axes": [{"name": "t", "type": "time", "unit": "millisecond"}, {"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "levels": 7, "datasetPaths": ["0", "1", "2", "3", "4", "5", "6"], "sizeT": 1, "sizeC": 1, "sizeZ": 2560, "sizeY": 1330, "sizeX": 1774, "dtype": "uint16", "fillValue": 0, "chunkShape": [1, 1, 128, 256, 256], "shardShape": null, "codecs": ["blosc"], "compressor": "blosc/zstd", "compressionLevel": 1, "voxelSizeX": 5.984, "voxelSizeY": 5.984, "voxelSizeZ": 8.0, "voxelSizeUnit": "micrometer", "channelNames": ["Channel:tile_000000_ch_488.zarr:0"], "omeroPresent": true, "presentChunks": 0, "presentChunksIsLowerBound": true, "metadataFiles": [".zgroup", ".zattrs", "0/.zarray", "6/.zarray"], "arrayMetadataMissing": ["1/.zarray", "2/.zarray", "3/.zarray", "4/.zarray", "5/.zarray"], "sidecars": []}}} +{"source": "exaSPIM_2026-08-04_10-27-41/SPIM/tile_000000_ch_488.ome.zarr/zarr.json", "sizeBytes": 1849, "mtime": 1789153861, "decoded": {"format": "ome-zarr", "summary": "OME-Zarr v3 (NGFF 0.5) multiscales, 6 levels, tczyx 1x1x1024x2660x3548 uint16, 20 x 15.04 x 15.04 micrometer, sharded, zstd", "fields": {"zarrFormat": 3, "ngffVersion": "0.5", "layout": "multiscales", "axes": [{"name": "t", "type": "time", "unit": "millisecond"}, {"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "levels": 6, "datasetPaths": ["0", "1", "2", "3", "4", "5"], "sizeT": 1, "sizeC": 1, "sizeZ": 1024, "sizeY": 2660, "sizeX": 3548, "dtype": "uint16", "fillValue": 0, "chunkShape": [1, 1, 128, 256, 256], "shardShape": [1, 1, 512, 512, 512], "codecs": ["sharding_indexed", "transpose", "bytes", "zstd"], "compressor": "zstd", "compressionLevel": 3, "voxelSizeX": 15.039999977980552, "voxelSizeY": 15.040000146851504, "voxelSizeZ": 20.0, "voxelSizeUnit": "micrometer", "channelNames": ["tile_000000_ch_488"], "omeroPresent": true, "presentChunks": 0, "presentChunksIsLowerBound": true, "metadataFiles": ["zarr.json", "0/zarr.json", "1/zarr.json", "2/zarr.json", "3/zarr.json", "4/zarr.json", "5/zarr.json"], "arrayMetadataMissing": [], "sidecars": []}}} diff --git a/fixtures/edge/voxel_run1_index.jsonl b/fixtures/edge/voxel_run1_index.jsonl new file mode 100644 index 0000000..63f5f74 --- /dev/null +++ b/fixtures/edge/voxel_run1_index.jsonl @@ -0,0 +1,5 @@ +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/manifest.json","sizeBytes":12515,"mtime":1792497656,"decoded":{"format":"voxel-manifest","summary":"voxel acquisition sim, completed, 4 volume(s), 4/4 dataset(s) completed, 2 channel(s) gfp/rfp, 488.0/561.0 nm, 8.0x","fields":{"availableLocations":4,"cameraDriver":"vxl.devices.camera.simulated.SimulatedCamera","cameraIds":["camera_1"],"channelEmissionNm":{"gfp":510,"rfp":600},"channelExcitationNm":{"gfp":488,"rfp":561},"channelIds":["gfp","rfp"],"channelLabels":{"gfp":"GFP","rfp":"RFP"},"completedDatasets":4,"completedVolumes":4,"createdAt":"2026-10-20T12:00:29.321395Z","datasetCount":4,"datasetStatusCounts":{"completed":4},"endedAt":"2026-10-20T12:00:56.503012Z","failure":null,"host":"Aarons-MacBook-Air.local","id":"26fdfb5c-31dd-40e6-8e8f-4f397fb53026","instrumentName":"sim","laserWavelengthsNm":[488,561],"objectiveMagnification":8,"operator":"scratch","output":{"batchZShards":1,"compression":"blosc.lz4","downscaleType":"gaussian","maxLevel":3,"shardZChunks":1,"targetShardGb":1},"planPoints":2,"profiles":["single_gfp","single_rfp"],"remoteRoot":null,"revision":15,"schemaVersion":"1.0","sensorHeightPx":128,"sensorPixelSizeUm":3.76,"sensorWidthPx":192,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","storePaths":["volumes/000001/single_gfp/gfp.ome.zarr","volumes/000002/single_rfp/rfp.ome.zarr","volumes/000003/single_gfp/gfp.ome.zarr","volumes/000004/single_rfp/rfp.ome.zarr"],"subjectId":null,"volumeCount":4,"volumeStatusCounts":{"completed":4},"volumes":[{"channel":"gfp","datasetStatus":"completed","format":"ome-zarr","index":1,"locationStatus":"available","profile":"single_gfp","storePath":"volumes/000001/single_gfp/gfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},{"channel":"rfp","datasetStatus":"completed","format":"ome-zarr","index":2,"locationStatus":"available","profile":"single_rfp","storePath":"volumes/000002/single_rfp/rfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},{"channel":"gfp","datasetStatus":"completed","format":"ome-zarr","index":3,"locationStatus":"available","profile":"single_gfp","storePath":"volumes/000003/single_gfp/gfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1},{"channel":"rfp","datasetStatus":"completed","format":"ome-zarr","index":4,"locationStatus":"available","profile":"single_rfp","storePath":"volumes/000004/single_rfp/rfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1}]}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000001/single_gfp/gfp.ome.zarr","sizeBytes":5545,"mtime":1792497636,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"gfp","channelLabel":"GFP","datasetStatus":"completed","emissionNm":510,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":488,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_gfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":1,"volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.29692,"endedAt":"2026-10-20T12:00:36.952690Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:32.655770Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000001/single_gfp/gfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000002/single_rfp/rfp.ome.zarr","sizeBytes":5545,"mtime":1792497643,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"rfp","channelLabel":"RFP","datasetStatus":"completed","emissionNm":600,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":561,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_rfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":2,"volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.215291,"endedAt":"2026-10-20T12:00:43.308542Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:39.093251Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000002/single_rfp/rfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000003/single_gfp/gfp.ome.zarr","sizeBytes":5545,"mtime":1792497650,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"gfp","channelLabel":"GFP","datasetStatus":"completed","emissionNm":510,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":488,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_gfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":3,"volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.18648,"endedAt":"2026-10-20T12:00:50.197589Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:46.011109Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000003/single_gfp/gfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000004/single_rfp/rfp.ome.zarr","sizeBytes":5545,"mtime":1792497656,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"rfp","channelLabel":"RFP","datasetStatus":"completed","emissionNm":600,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":561,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_rfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":4,"volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.162044,"endedAt":"2026-10-20T12:00:56.457014Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:52.294970Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000004/single_rfp/rfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} diff --git a/src/scigantic_bil/edge.py b/src/scigantic_bil/edge.py index 6aac3da..59eac84 100644 --- a/src/scigantic_bil/edge.py +++ b/src/scigantic_bil/edge.py @@ -16,8 +16,9 @@ box wrote while watching the acquisition and fills in BIL's Image and Instrument submission sheets from the headers it decoded (SmartSPIM metadata.txt, Zeiss CZI, Luxendo and mesoSPIM sidecars, lattice Settings.txt, -BigDataViewer XML, Micro-Manager metadata, OME-TIFF, Imaris, KLB, ND2). On -2026-10-17, 849 of 1,004 light-sheet records on BIL had an Instrument +BigDataViewer XML, Micro-Manager metadata, OME-TIFF, Imaris, KLB, ND2, +OME-Zarr stores and the ``manifest.json`` of an Allen Institute ``voxel`` +acquisition). On 2026-10-17, 849 of 1,004 light-sheet records on BIL had an Instrument block and about half of those had objective, detector and wavelength fields; the rest were left blank by hand. These are exactly the values the microscope wrote beside its planes. @@ -26,12 +27,20 @@ (``stepsizex``, ``microscopemanufacturerandmodel`` and so on), so a draft row can be pasted into the submission spreadsheet column for column. + +The prior's ``bil-image`` fields already cover what an OME-Zarr record on +the box can say about itself: ``stepSizeXUm``/``YUm``/``ZUm`` against the +store's level-0 scale in micrometres, ``channels`` against ``sizeC``, +``slices`` and ``sizeZ`` against ``sizeZ``, ``sizeX``/``sizeY`` against the +array shape, ``dimensionOrder`` against the axes names. Nothing OME-Zarr +specific is added to the prior. """ from __future__ import annotations import base64 import json +import posixpath from pathlib import Path from typing import Any, Iterable, Iterator @@ -188,13 +197,30 @@ def read_index(path: str | Path) -> Iterator[dict[str, Any]]: yield rec +# Micrometres per unit, keyed by the spellings seen in BigDataViewer XML, +# OME-TIFF (symbols) and OME-NGFF axes (UDUNITS-2 names: "micrometer", +# "nanometer", "millimeter"). British spellings and plurals are folded in +# before lookup. An unknown or missing unit is taken as micrometres, the +# convention every other decoder here already relies on. +_UM_PER_UNIT = { + "pm": 1e-6, "picometer": 1e-6, + "nm": 1e-3, "nanometer": 1e-3, + "um": 1.0, "\u00b5m": 1.0, "\u03bcm": 1.0, "micrometer": 1.0, "micron": 1.0, + "mm": 1e3, "millimeter": 1e3, + "cm": 1e4, "centimeter": 1e4, + "m": 1e6, "meter": 1e6, + "angstrom": 1e-4, "\u00e5": 1e-4, "\u212b": 1e-4, +} + + def _um(value: Any, unit: Any = None) -> float | None: v = _num(value) if v is None: return None - u = (str(unit) if unit else "").strip().lower() - scale = {"nm": 1e-3, "mm": 1e3, "m": 1e6, "cm": 1e4, "pm": 1e-6}.get(u, 1.0) - return v * scale + u = (str(unit) if unit else "").strip().lower().replace("metre", "meter") + if u not in _UM_PER_UNIT and len(u) > 3 and u.endswith("s"): + u = u[:-1] # microns, micrometers; never "ms" + return v * _UM_PER_UNIT.get(u, 1.0) def _fmt(v: float | None) -> str: @@ -263,6 +289,15 @@ def _image_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: elif fmt == "nd2": sx = sy = _num(f.get("pixelSizeUm")) xsize, ysize = _int(f.get("imageWidth")), _int(f.get("imageLength")) + elif fmt == "ome-zarr": + # voxelSize* is the level-0 scale transform in voxelSizeUnit (an + # NGFF axis unit name, "micrometer" in every store seen so far). + u = f.get("voxelSizeUnit") + sx, sy, sz = _um(f.get("voxelSizeX"), u), _um(f.get("voxelSizeY"), u), _um(f.get("voxelSizeZ"), u) + channels, slices = _int(f.get("sizeC")), _int(f.get("sizeZ")) + xsize, ysize, zsize = _int(f.get("sizeX")), _int(f.get("sizeY")), _int(f.get("sizeZ")) + names = [str(a.get("name") or "") for a in (f.get("axes") or []) if isinstance(a, dict)] + order = "".join(names).upper() or None else: return row row["stepsizex"], row["stepsizey"], row["stepsizez"] = _fmt(sx), _fmt(sy), _fmt(sz) @@ -344,13 +379,36 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("numericalAperture"))) row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["detectormodel"] = _text(f.get("cameraName")) or "" + elif fmt == "voxel-manifest": + # voxel is a generic acquisition engine; its template instrument is + # a simulator. The manifest says light-sheet only when the + # instrument's name does (exaSPIM, diSPIM, ...), so the type is left + # blank otherwise rather than guessed from the engine's origin. + name = _text(f.get("instrumentName")) or "" + row["microscopetype"] = "Light-sheet" if _names_light_sheet(name) else "" + row["microscopemanufacturerandmodel"] = name + row["objectivename"] = _text(f.get("objectiveName")) or "" + row["objectivena"] = _fmt(_num(f.get("objectiveNA"))) + row["objectivemagnification"] = _fmt(_num(f.get("objectiveMagnification"))) + row["detectormodel"] = _text(f.get("cameraModel")) or "" + lines = [_fmt(_num(v)) for v in (f.get("laserWavelengthsNm") or []) if _num(v)] + row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") + row["illuminationtypes"] = "laser" if lines else "" + em_table = f.get("channelEmissionNm") + em = [_fmt(_num(v)) for v in (em_table.values() if isinstance(em_table, dict) else []) if _num(v)] + row["detectionwavelength"] = ", ".join(em) + (" nm" if em else "") return row +def _names_light_sheet(name: str) -> bool: + flat = name.lower().replace("-", "").replace("_", "").replace(" ", "") + return any(tag in flat for tag in ("lightsheet", "spim", "lsfm")) + + # Formats ranked by how much of the submission they can fill; the best # available one supplies each column, later ones only fill blanks. -_PREFERENCE = ("smartspim-metadata", "czi", "luxendo-stack", "mesospim-meta", "lls-settings", "bdv-spimdata", - "micromanager-metadata", "ome-tiff", "imaris", "klb", "nd2") +_PREFERENCE = ("smartspim-metadata", "czi", "luxendo-stack", "mesospim-meta", "lls-settings", "voxel-manifest", + "bdv-spimdata", "micromanager-metadata", "ome-zarr", "ome-tiff", "imaris", "klb", "nd2") def _dir_of(source: str) -> str: @@ -359,6 +417,74 @@ def _dir_of(source: str) -> str: _PLANE_FORMATS = frozenset({"png", "tiff", "dcimg"}) +_STORE_METADATA = frozenset({"zarr.json", ".zattrs", ".zgroup", ".zarray"}) + + +def _store_of(source: str) -> str: + """The store directory an ``ome-zarr`` record describes: the deepest + path component ending in ``.zarr``, so a record keyed on the store's + ``zarr.json`` or ``.zattrs`` and one keyed on the directory agree. A + source with no such component is its own store (or, for a bare + metadata file, its directory).""" + s = source.replace("\\", "/").rstrip("/") + parts = s.split("/") + for i in range(len(parts), 0, -1): + if parts[i - 1].lower().endswith(".zarr"): + return "/".join(parts[:i]) + if parts[-1] in _STORE_METADATA: + return "/".join(parts[:-1]) + return s + + +def _store_name(store: str) -> str: + base = store.rsplit("/", 1)[-1] + low = base.lower() + for suffix in (".ome.zarr", ".zarr"): + if low.endswith(suffix): + return base[: -len(suffix)] + return base + + +def _voxel_volume(store: str) -> tuple[str, str] | None: + """``(acquisition root, volume directory)`` for a store laid out as + voxel writes it, ``/volumes/NNNNNN//.ome.zarr``; + ``None`` for any other path.""" + parts = store.split("/") + for i in range(len(parts) - 2): + if parts[i] == "volumes" and parts[i + 1].isdigit(): + return "/".join(parts[:i]), "/".join(parts[: i + 2]) + return None + + +def _manifest_candidates(store: str, f: dict[str, Any]) -> list[str]: + """Where a store's ``manifest.json`` sidecar would sit in the index: + the sidecar resolved against the store directory (the decoder writes + sidecars relative to the store) and the sidecar as written.""" + out: list[str] = [] + for sc in f.get("sidecars") or []: + s = str(sc).replace("\\", "/") + if s.rsplit("/", 1)[-1] != "manifest.json": + continue + if not s.startswith("/"): + out.append(posixpath.normpath(posixpath.join(store, s))) + out.append(s) + return out + + +def _store_home(source: str, f: dict[str, Any], manifests: dict[str, str], store_paths: list[tuple[str, str]]) -> str: + """The acquisition directory an ``ome-zarr`` record belongs to; the + rule is spelled out on :func:`submission_drafts`.""" + store = _store_of(source) + for cand in _manifest_candidates(store, f): + if cand in manifests: + return manifests[cand] + for path, home in store_paths: + if store in (path, posixpath.normpath(posixpath.join(home, path)) if home else path) or path.endswith("/" + store): + return home + vol = _voxel_volume(store) + if vol is not None: + return vol[0] + return _dir_of(store) def submission_drafts(records: Iterable[dict[str, Any]] | str | Path) -> dict[str, dict[str, Any]]: @@ -371,16 +497,57 @@ def submission_drafts(records: Iterable[dict[str, Any]] | str | Path) -> dict[st plain TIFF) are counted under the deepest such directory that contains them. Returns ``{directory: draft}`` with the same draft shape as :func:`submission_draft`. + + An ``ome-zarr`` record is keyed on its store (the deepest path + component ending in ``.zarr``, whether the record names the directory + or its ``zarr.json``) and goes to the first of these that applies: + + 1. the directory of a ``voxel-manifest`` record that one of the store's + ``sidecars`` points at (``../../../manifest.json`` resolved against + the store, or the sidecar as written); + 2. the directory of a ``voxel-manifest`` record whose ``storePaths`` + lists the store (relative to the manifest's directory, as the + decoder writes them, or equal to the store's path); + 3. the path prefix before ``/volumes/NNNNNN/``, voxel's layout, when + no manifest has been indexed yet; + 4. the store's parent directory, like any other file. The public + AIND layout ``SPIM.ome.zarr/tile_NNNNNN_ch_NNN.zarr`` therefore + collapses to one draft per ``SPIM.ome.zarr``. + + A voxel acquisition is one BIL submission made of many single-channel + stores (one per volume and channel). Within such a group the Image + row's ``channels`` is the number of distinct channels across the + volumes (each store's ``fields.manifest.channelId``, or its + ``.ome.zarr`` name when the decoder paired no manifest), not + the ``sizeC`` of one store (always 1); volumes are counted the same + way (``fields.manifest.volumeIndex``, else the ``volumes/NNNNNN`` + directory). The draft's ``acquisition`` block carries what BIL's + sheets have no column for: store and volume counts, channel names, + the manifest's status, revision and timestamps. """ recs = list(read_index(records)) if isinstance(records, (str, Path)) else list(records) + manifests: dict[str, str] = {} + store_paths: list[tuple[str, str]] = [] + for r in recs: + d = r.get("decoded") or {} + if str(d.get("format", "")) != "voxel-manifest": + continue + src = str(r.get("source", "")).replace("\\", "/") + manifests[src] = _dir_of(src) + for p in (d.get("fields") or {}).get("storePaths") or []: + store_paths.append((str(p).replace("\\", "/").rstrip("/"), manifests[src])) groups: dict[str, list[dict[str, Any]]] = {} planes: list[dict[str, Any]] = [] for r in recs: - fmt = str((r.get("decoded") or {}).get("format", "")) + d = r.get("decoded") or {} + fmt = str(d.get("format", "")) + src = str(r.get("source", "")) if fmt in _PLANE_FORMATS: planes.append(r) + elif fmt == "ome-zarr": + groups.setdefault(_store_home(src, d.get("fields") or {}, manifests, store_paths), []).append(r) else: - groups.setdefault(_dir_of(str(r.get("source", ""))), []).append(r) + groups.setdefault(_dir_of(src), []).append(r) if not groups: groups[""] = [] dirs = sorted(groups, key=len, reverse=True) @@ -402,11 +569,13 @@ def submission_draft(records: Iterable[dict[str, Any]] | str | Path, cohort: str instruments into one deposit. Returns ``{"image": {...}, "instrument": {...}, "sources": [...], - "formats": {...}}`` with every BIL column present (blank where - nothing on the box says), the records each value came from, and a - per-format count of what was read. Plane files (PNG, plain TIFF) are - counted, not used: they say width and height, which BIL does not ask - for. + "formats": {...}, "acquisition": {...}}`` with every BIL column + present (blank where nothing on the box says), the records each value + came from, a per-format count of what was read, and, for OME-Zarr + stores and voxel manifests, the counts and status BIL has no column + for (empty otherwise; see :func:`submission_drafts`). Plane files + (PNG, plain TIFF) are counted, not used: they say width and height, + which BIL does not ask for. """ drafts = submission_drafts(records) if cohort is not None: @@ -449,4 +618,51 @@ def _draft(recs: list[dict[str, Any]]) -> dict[str, Any]: # a count of files is a lower bound on slices x channels x tiles, # reported as files rather than guessed into slices. image["files"] = str(planes) - return {"image": image, "instrument": instrument, "sources": sources, "formats": counts} + acquisition = _acquisition(by_format.get("ome-zarr", []), by_format.get("voxel-manifest", []), image, sources) + return {"image": image, "instrument": instrument, "sources": sources, "formats": counts, "acquisition": acquisition} + + +_MANIFEST_KEYS = (("status", "status"), ("revision", "revision"), ("startedAt", "startedAt"), ("endedAt", "endedAt"), + ("volumesPlanned", "volumeCount"), ("volumesCompleted", "completedVolumes")) + + +def _acquisition(stores: list[dict[str, Any]], manifests: list[dict[str, Any]], + image: dict[str, str], sources: list[str]) -> dict[str, Any]: + """The ``acquisition`` block of a draft, and the one Image column it + corrects: ``channels`` of a voxel acquisition, counted across its + single-channel stores.""" + out: dict[str, Any] = {} + if not stores and not manifests: + return out + out["stores"] = len(stores) + first_store_of: dict[str, str] = {} + volumes: set[str] = set() + for r in stores: + store = _store_of(str(r.get("source", ""))) + view = ((r.get("decoded") or {}).get("fields") or {}).get("manifest") + view = view if isinstance(view, dict) else {} + vol = _voxel_volume(store) + if not view and vol is None: + continue + volumes.add(str(view.get("volumeIndex")) if view.get("volumeIndex") is not None else (vol[1] if vol else store)) + first_store_of.setdefault(_text(view.get("channelId")) or _store_name(store), store) + if volumes: + out["volumes"] = len(volumes) + out["channels"] = sorted(first_store_of) + if manifests: + m = manifests[0] + f = (m.get("decoded") or {}).get("fields") or {} + out["manifest"] = str(m.get("source", "")) + for k_out, k_in in _MANIFEST_KEYS: + if f.get(k_in) is not None: + out[k_out] = f[k_in] + if "channels" not in out and f.get("channelIds"): + out["channels"] = [str(c) for c in f["channelIds"]] + if volumes: + image["channels"] = str(len(first_store_of)) + for store in first_store_of.values(): + if store not in sources: + sources.append(store) + elif out.get("channels") and not image["channels"]: + image["channels"] = str(len(out["channels"])) + return out diff --git a/tests/test_edge.py b/tests/test_edge.py index bf74597..f183e11 100644 --- a/tests/test_edge.py +++ b/tests/test_edge.py @@ -168,3 +168,149 @@ def test_cli_subcommands(tmp_path: Path, capsys) -> None: # type: ignore[no-unt assert out["run"]["instrument"]["objectivename"] == "LCT 3.6x" assert main(["edge-prior", "ace-cup-eel", str(tmp_path / "p.b64")]) == 0 assert (tmp_path / "p.b64").stat().st_size > 100 + + +# OME-Zarr stores and voxel manifests. voxel_run1_index.jsonl is the +# scigantic-edge decoder's own index.jsonl over a real headless run of the +# Allen Institute voxel engine's simulated instrument (2 XY tiles x 2 +# single-channel profiles, 10 frames of 128 x 192 uint16, 2026-10-20): one +# voxel-manifest record and four ome-zarr records, each carrying the +# manifest's view of that store under fields.manifest. The exaSPIM records +# are hand-written to the same contract from two public tiles' metadata on +# s3://aind-open-data (Zarr v2 / NGFF 0.4 and Zarr v3 / NGFF 0.5, tczyx). +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "edge" +VOXEL_RUN1 = FIXTURES / "voxel_run1_index.jsonl" +EXASPIM_PUBLIC = FIXTURES / "exaspim_public_index.jsonl" + + +def _records(path: Path) -> list[dict]: # type: ignore[type-arg] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def _voxel_parts() -> tuple[dict, list[dict]]: # type: ignore[type-arg] + recs = _records(VOXEL_RUN1) + manifests = [r for r in recs if r["decoded"]["format"] == "voxel-manifest"] + stores = [r for r in recs if r["decoded"]["format"] == "ome-zarr"] + assert len(manifests) == 1 and len(stores) == 4 + return manifests[0], stores + + +def test_voxel_acquisition_is_one_submission() -> None: + drafts = submission_drafts(VOXEL_RUN1) + assert len(drafts) == 1 + (root,) = drafts + assert root.endswith("/run1/store/run1") # the directory holding manifest.json + d = submission_draft(VOXEL_RUN1) + img, inst, acq = d["image"], d["instrument"], d["acquisition"] + assert (img["stepsizex"], img["stepsizey"], img["stepsizez"]) == ("0.47", "0.47", "1") + assert img["channels"] == "2" # gfp + rfp across volumes, not the sizeC=1 of one store + assert (img["slices"], img["zsize"], img["ysize"], img["xsize"]) == ("10", "10", "128", "192") + assert img["dimensionorder"] == "CZYX" + assert acq["status"] == "completed" and acq["revision"] == 15 + assert acq["stores"] == 4 and acq["volumes"] == 4 and acq["volumesPlanned"] == 4 and acq["volumesCompleted"] == 4 + assert acq["channels"] == ["gfp", "rfp"] and acq["manifest"].endswith("/run1/store/run1/manifest.json") + assert acq["startedAt"] < acq["endedAt"] + # The template instrument is a simulator: no light-sheet claim, model as named, + # no objective name/NA or camera model in the record, so those stay blank. + assert inst["microscopetype"] == "" and inst["microscopemanufacturerandmodel"] == "sim" + assert inst["objectivemagnification"] == "8" and inst["objectivena"] == "" and inst["objectivename"] == "" + assert inst["illuminationwavelength"] == "488, 561 nm" and inst["illuminationtypes"] == "laser" + assert inst["detectionwavelength"] == "510, 600 nm" + assert inst["detectormodel"] == "" and inst["detectortype"] == "" + assert d["formats"] == {"voxel-manifest": 1, "ome-zarr": 4} + assert d["sources"][0].endswith("/manifest.json") + assert {s.rsplit("/", 1)[-1] for s in d["sources"][1:]} == {"gfp.ome.zarr", "rfp.ome.zarr"} + + +def test_voxel_stores_group_under_their_manifest_by_each_rule() -> None: + manifest, stores = _voxel_parts() + root = manifest["source"].rsplit("/", 1)[0] + # Rule 1: the sidecar ../../../../manifest.json resolved against the store lands on the + # manifest record, also when the record is keyed on the store's zarr.json. + keyed = json.loads(json.dumps(stores)) + for r in keyed: + r["source"] += "/zarr.json" + assert set(submission_drafts([manifest] + keyed)) == {root} + assert submission_draft([manifest] + keyed)["image"]["channels"] == "2" + # Rule 2: no sidecars; the manifest's storePaths (relative to its directory) name the stores. + no_sidecar = json.loads(json.dumps(stores)) + for r in no_sidecar: + r["decoded"]["fields"]["sidecars"] = [] + assert set(submission_drafts([manifest] + no_sidecar)) == {root} + stale = json.loads(json.dumps(manifest)) + stale["decoded"]["fields"]["storePaths"] = [] + stale["source"] = root + "/elsewhere/manifest.json" + # Neither rule 1 nor 2 pairs the stores with a manifest in another directory: rule 3 keeps + # them together on their own, under the /volumes/ prefix, and the stray manifest is its own draft. + assert set(submission_drafts([stale] + no_sidecar)) == {root, root + "/elsewhere"} + # Rule 3 alone: no manifest record at all yet. Channel ids still come from each store's + # own manifest view, the counts from the stores. + only_stores = submission_drafts(no_sidecar) + assert set(only_stores) == {root} + d = only_stores[root] + assert d["image"]["channels"] == "2" and d["image"]["slices"] == "10" + assert d["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"]} + assert d["instrument"]["microscopemanufacturerandmodel"] == "" + # Without the per-store manifest view either, the .ome.zarr names and volume + # directories carry the same counts. + bare = json.loads(json.dumps(no_sidecar)) + for r in bare: + r["decoded"]["fields"].pop("manifest", None) + assert submission_draft(bare)["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"]} + # A voxel run under a different root is a different draft, not a blend. + other = json.loads(json.dumps(no_sidecar[:1])) + other[0]["source"] = other[0]["source"].replace("/run1/volumes/", "/run2/volumes/") + assert len(submission_drafts(no_sidecar + other)) == 2 + + +def test_voxel_manifest_alone_seeds_the_draft_before_stores_arrive() -> None: + manifest, _ = _voxel_parts() + d = submission_draft([manifest]) + assert d["image"]["channels"] == "2" and d["image"]["stepsizex"] == "" and d["image"]["slices"] == "" + assert d["acquisition"]["status"] == "completed" and d["acquisition"]["stores"] == 0 + assert "volumes" not in d["acquisition"] and d["acquisition"]["channels"] == ["gfp", "rfp"] + assert d["instrument"]["illuminationwavelength"] == "488, 561 nm" + + +def test_voxel_manifest_light_sheet_only_when_named() -> None: + manifest = json.loads(json.dumps(_voxel_parts()[0])) + manifest["decoded"]["fields"]["instrumentName"] = "exaSPIM-1" + inst = submission_draft([manifest])["instrument"] + assert inst["microscopetype"] == "Light-sheet" and inst["microscopemanufacturerandmodel"] == "exaSPIM-1" + manifest["decoded"]["fields"]["instrumentName"] = "confocal-2" + assert submission_draft([manifest])["instrument"]["microscopetype"] == "" + + +def test_public_exaspim_stores_map_without_a_manifest() -> None: + drafts = submission_drafts(EXASPIM_PUBLIC) + assert set(drafts) == {"exaSPIM_851474_2026-07-10_15-23-22/SPIM.ome.zarr", "exaSPIM_2026-08-04_10-27-41/SPIM"} + v2 = drafts["exaSPIM_851474_2026-07-10_15-23-22/SPIM.ome.zarr"] + img = v2["image"] + assert (img["stepsizex"], img["stepsizey"], img["stepsizez"]) == ("5.984", "5.984", "8") + assert (img["channels"], img["slices"], img["ysize"], img["xsize"]) == ("1", "2560", "1330", "1774") + assert img["dimensionorder"] == "TCZYX" + assert all(v == "" for v in v2["instrument"].values()) + assert v2["acquisition"] == {"stores": 1} and v2["formats"] == {"ome-zarr": 1} + v3 = drafts["exaSPIM_2026-08-04_10-27-41/SPIM"] + assert (v3["image"]["stepsizex"], v3["image"]["stepsizez"], v3["image"]["slices"]) == ("15.04", "20", "1024") + assert v3["sources"] == ["exaSPIM_2026-08-04_10-27-41/SPIM/tile_000000_ch_488.ome.zarr/zarr.json"] + + +def test_ome_zarr_voxel_size_units_reach_micrometres() -> None: + from scigantic_bil.edge import _um + + assert _um(0.47, "micrometer") == 0.47 and _um(0.47, "um") == 0.47 + assert _um(0.47, "\u00b5m") == 0.47 and _um(0.47, "\u03bcm") == 0.47 # micro sign, Greek mu + assert _um(0.47, "microns") == 0.47 and _um(0.47, "micrometres") == 0.47 + assert _um(470, "nanometer") == pytest.approx(0.47) and _um(470, "nm") == pytest.approx(0.47) + assert _um(0.001, "millimeter") == pytest.approx(1.0) and _um(0.001, "mm") == pytest.approx(1.0) + assert _um(2, None) == 2.0 and _um(None, "micrometer") is None + rec = json.loads(json.dumps(_records(EXASPIM_PUBLIC)[0])) + f = rec["decoded"]["fields"] + f["voxelSizeUnit"], f["voxelSizeX"], f["voxelSizeY"], f["voxelSizeZ"] = "nanometer", 5984, 5984, 8000 + img = submission_draft([rec])["image"] + assert (img["stepsizex"], img["stepsizez"]) == ("5.984", "8") + + +def test_draft_without_stores_has_an_empty_acquisition_block() -> None: + assert submission_draft([SMARTSPIM])["acquisition"] == {} From e697565717a23ed0fd26897202504f6f26849a4a Mon Sep 17 00:00:00 2001 From: aaronkanzer <36093535+aaronkanzer@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:44:34 -0400 Subject: [PATCH 2/2] edge: keep the latest record per source, never blend stores, survive malformed records Found by running the scigantic-edge binary over its own fixture stores and feeding the index through the bridge, then at scale and under fuzzing. - index.jsonl is an append log. In -watch mode a store re-records as its shards land and a voxel manifest.json re-records on every status change, so read_index now yields only the last line per source, as the box's own resume does. An 8,117-line watch log of a 200-volume, 4-channel run drafted 8,000 stores, 40 slices (the first partial record) and status "running" at revision 0; it now drafts 800 stores, 400 slices, "completed", rev 49. - A plain container group (AIND's SPIM.ome.zarr) joins the draft of the stores it holds. It used to start an empty draft of its own, so submission_draft() raised on every public AIND v2 acquisition. - One reference store (has a shape, most complete, finest voxel) supplies the whole Image row. Filling blanks from the next store paired the 561 nm store's 30.08 um voxel with the 488 nm store's 3548 px width on the 2026-08-04 public exaSPIM acquisition. Disagreements across stores are reported in acquisition.mismatch, and store states in storeStates. - AIND's tile_..._ch_ store names count tiles and channels. - An OME-Zarr scale with no unit, or a non-length unit, leaves the step sizes blank instead of calling it micrometres. - Every record is normalised on entry; list fields, NaN and infinity are type-checked. 30,000 random malformed indexes raise nothing. --- src/scigantic_bil/edge.py | 175 +++++++++++++++++++++++++++++++++----- tests/test_edge.py | 95 ++++++++++++++++++++- 2 files changed, 243 insertions(+), 27 deletions(-) diff --git a/src/scigantic_bil/edge.py b/src/scigantic_bil/edge.py index 59eac84..493d913 100644 --- a/src/scigantic_bil/edge.py +++ b/src/scigantic_bil/edge.py @@ -40,7 +40,9 @@ import base64 import json +import math import posixpath +import re from pathlib import Path from typing import Any, Iterable, Iterator @@ -78,7 +80,7 @@ def _num(value: Any) -> float | None: v = float(str(value).strip()) except ValueError: return None - return v if v == v else None # NaN check + return v if math.isfinite(v) else None # NaN and +-inf are not values def _int(value: Any) -> int | None: @@ -86,6 +88,11 @@ def _int(value: Any) -> int | None: return int(v) if v is not None and v > 0 else None +def _list(value: Any) -> list[Any]: + """``value`` if the decoder wrote a list there, else an empty list.""" + return value if isinstance(value, list) else [] + + def _text(value: Any) -> str | None: s = str(value).strip() if value is not None else "" return s or None @@ -180,10 +187,18 @@ def write_prior(detail: DatasetDetail | str, path: str | Path) -> Path: def read_index(path: str | Path) -> Iterator[dict[str, Any]]: """Yield the records of an edge box's ``index.jsonl`` (or a directory - holding one), tolerating a torn last line.""" + holding one), tolerating a torn last line. + + ``index.jsonl`` is an append log: a box in ``-watch`` mode writes a new + line each time a source changes (an OME-Zarr store re-records as its + shards land, a voxel ``manifest.json`` is rewritten on every status + change). Only the last line for each source is yielded, in the order + sources first appeared, which is what the box's own resume reads too. + """ p = Path(path) if p.is_dir(): p = p / "index.jsonl" + recs: list[dict[str, Any]] = [] with p.open("r", encoding="utf-8") as fh: for line in fh: line = line.strip() @@ -194,7 +209,41 @@ def read_index(path: str | Path) -> Iterator[dict[str, Any]]: except json.JSONDecodeError: continue if isinstance(rec, dict) and isinstance(rec.get("decoded"), dict): - yield rec + recs.append(rec) + yield from _latest(recs) + + +def _clean(r: Any) -> dict[str, Any] | None: + """A record with a dict ``decoded``, a dict ``fields`` and string + ``format`` and ``source``, or ``None`` for one that is not a record. + Every reader below can then index into it without checking types.""" + if not isinstance(r, dict) or not isinstance(r.get("decoded"), dict): + return None + d = dict(r["decoded"]) + d["format"] = str(d.get("format") or "") + d["fields"] = d["fields"] if isinstance(d.get("fields"), dict) else {} + src = r.get("source") + return dict(r, decoded=d, source=src if isinstance(src, str) else "") + + +def _latest(records: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + """The last record for each ``source``, in first-seen order. Records + with no source are kept as they are.""" + by_source: dict[str, dict[str, Any]] = {} + loose: list[tuple[int, dict[str, Any]]] = [] + order: dict[str, int] = {} + for i, raw in enumerate(records): + r = _clean(raw) + if r is None: + continue + src = str(r.get("source", "")).replace("\\", "/").rstrip("/") + if not src: + loose.append((i, r)) + continue + order.setdefault(src, i) + by_source[src] = r + keyed = [(order[k], v) for k, v in by_source.items()] + loose + return [r for _, r in sorted(keyed, key=lambda t: t[0])] # Micrometres per unit, keyed by the spellings seen in BigDataViewer XML, @@ -223,6 +272,19 @@ def _um(value: Any, unit: Any = None) -> float | None: return v * _UM_PER_UNIT.get(u, 1.0) +def _um_declared(value: Any, unit: Any) -> float | None: + """Like :func:`_um`, but ``None`` when the unit is missing or not a + length. OME-NGFF makes an axis unit optional, and a scale with no unit + is in no known unit at all, so it is not reported as micrometres.""" + v = _num(value) + u = (str(unit) if unit else "").strip().lower().replace("metre", "meter") + if u not in _UM_PER_UNIT and len(u) > 3 and u.endswith("s"): + u = u[:-1] + if v is None or u not in _UM_PER_UNIT: + return None + return v * _UM_PER_UNIT[u] + + def _fmt(v: float | None) -> str: if v is None: return "" @@ -275,7 +337,7 @@ def _image_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: sx, sy, sz = _num(f.get("voxelSizeXUm")), _num(f.get("voxelSizeYUm")), _num(f.get("voxelSizeZUm")) slices = _int(f.get("sizeZ")) xsize, ysize, zsize = _int(f.get("sizeX")), _int(f.get("sizeY")), _int(f.get("sizeZ")) - on = f.get("lasersOnNm") or [] + on = _list(f.get("lasersOnNm")) channels = len(on) if on else None elif fmt == "lls-settings": sz = _num(f.get("stepUm")) @@ -292,11 +354,12 @@ def _image_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: elif fmt == "ome-zarr": # voxelSize* is the level-0 scale transform in voxelSizeUnit (an # NGFF axis unit name, "micrometer" in every store seen so far). + # A missing or non-length unit leaves the step sizes blank. u = f.get("voxelSizeUnit") - sx, sy, sz = _um(f.get("voxelSizeX"), u), _um(f.get("voxelSizeY"), u), _um(f.get("voxelSizeZ"), u) + sx, sy, sz = (_um_declared(f.get(k), u) for k in ("voxelSizeX", "voxelSizeY", "voxelSizeZ")) channels, slices = _int(f.get("sizeC")), _int(f.get("sizeZ")) xsize, ysize, zsize = _int(f.get("sizeX")), _int(f.get("sizeY")), _int(f.get("sizeZ")) - names = [str(a.get("name") or "") for a in (f.get("axes") or []) if isinstance(a, dict)] + names = [str(a.get("name") or "") for a in _list(f.get("axes")) if isinstance(a, dict)] order = "".join(names).upper() or None else: return row @@ -316,7 +379,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivename"] = _text(f.get("objective")) or "" row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["objectiveimmersion"] = _text(f.get("immersion")) or "" - lines = [str(int(_num(w) or 0)) for w in (f.get("channelWavelengthsNm") or []) if _num(w)] + lines = [str(int(_num(w) or 0)) for w in _list(f.get("channelWavelengthsNm")) if _num(w)] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" elif fmt == "czi": @@ -328,11 +391,11 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("objectiveNA"))) row["objectivemagnification"] = _fmt(_num(f.get("objectiveMagnification"))) row["objectiveimmersion"] = _text(f.get("objectiveImmersion")) or "" - dets = [d for d in (f.get("detectorModels") or []) if d] + dets = [d for d in _list(f.get("detectorModels")) if d] row["detectormodel"] = ", ".join(str(d) for d in dets) row["detectortype"] = "CMOS" if dets else "" - ex = [_fmt(_num(v)) for v in (f.get("channelExcitationNm") or []) if _num(v)] - em = [_fmt(_num(v)) for v in (f.get("channelEmissionNm") or []) if _num(v)] + ex = [_fmt(_num(v)) for v in _list(f.get("channelExcitationNm")) if _num(v)] + em = [_fmt(_num(v)) for v in _list(f.get("channelEmissionNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(ex) + (" nm" if ex else "") row["detectionwavelength"] = ", ".join(em) + (" nm" if em else "") elif fmt == "imaris": @@ -340,7 +403,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["microscopetype"] = "Light-sheet" if "lightsheet" in mode.lower().replace(" ", "") else mode row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["objectivena"] = _fmt(_num(f.get("numericalAperture"))) - ex = [_fmt(_num(v)) for v in (f.get("excitationWavelengthsNm") or []) if _num(v)] + ex = [_fmt(_num(v)) for v in _list(f.get("excitationWavelengthsNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(ex) + (" nm" if ex else "") elif fmt == "bdv-spimdata": row["microscopetype"] = "Light-sheet" @@ -351,7 +414,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["detectortype"] = "CMOS" if f.get("cameraType") else "" row["detectormodel"] = _text(f.get("cameraType")) or "" - lines = [str(int(_num(str(w).split()[0]) or 0)) for w in (f.get("lasers") or []) if _num(str(w).split()[0])] + lines = [str(int(_num(str(w).split()[0]) or 0)) for w in _list(f.get("lasers")) if _num(str(w).split()[0])] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" elif fmt == "luxendo-stack": @@ -362,7 +425,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("numericalAperture"))) row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["objectiveimmersion"] = _text(f.get("objectiveImmersion")) or "" - on = [_fmt(_num(v)) for v in (f.get("lasersOnNm") or []) if _num(v)] + on = [_fmt(_num(v)) for v in _list(f.get("lasersOnNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(on) + (" nm" if on else "") row["illuminationtypes"] = "laser" if on else "" elif fmt == "lls-settings": @@ -370,7 +433,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["microscopemanufacturerandmodel"] = "Lattice light-sheet (Janelia design)" row["detectortype"] = "CMOS" if f.get("cameraModel") else "" row["detectormodel"] = _text(f.get("cameraModel")) or "" - lines = [_fmt(_num(v)) for v in (f.get("laserLinesNm") or []) if _num(v)] + lines = [_fmt(_num(v)) for v in _list(f.get("laserLinesNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" elif fmt == "nd2": @@ -391,7 +454,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("objectiveNA"))) row["objectivemagnification"] = _fmt(_num(f.get("objectiveMagnification"))) row["detectormodel"] = _text(f.get("cameraModel")) or "" - lines = [_fmt(_num(v)) for v in (f.get("laserWavelengthsNm") or []) if _num(v)] + lines = [_fmt(_num(v)) for v in _list(f.get("laserWavelengthsNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" em_table = f.get("channelEmissionNm") @@ -461,7 +524,7 @@ def _manifest_candidates(store: str, f: dict[str, Any]) -> list[str]: the sidecar resolved against the store directory (the decoder writes sidecars relative to the store) and the sidecar as written.""" out: list[str] = [] - for sc in f.get("sidecars") or []: + for sc in _list(f.get("sidecars")): s = str(sc).replace("\\", "/") if s.rsplit("/", 1)[-1] != "manifest.json": continue @@ -475,6 +538,11 @@ def _store_home(source: str, f: dict[str, Any], manifests: dict[str, str], store """The acquisition directory an ``ome-zarr`` record belongs to; the rule is spelled out on :func:`submission_drafts`.""" store = _store_of(source) + if f.get("layout") == "group": + # A plain container group (AIND's SPIM.ome.zarr holding one store + # per tile and channel) is the acquisition its member stores sit + # in, so it joins their draft instead of starting one of its own. + return store for cand in _manifest_candidates(store, f): if cand in manifests: return manifests[cand] @@ -525,7 +593,7 @@ def submission_drafts(records: Iterable[dict[str, Any]] | str | Path) -> dict[st sheets have no column for: store and volume counts, channel names, the manifest's status, revision and timestamps. """ - recs = list(read_index(records)) if isinstance(records, (str, Path)) else list(records) + recs = list(read_index(records)) if isinstance(records, (str, Path)) else _latest(records) manifests: dict[str, str] = {} store_paths: list[tuple[str, str]] = [] for r in recs: @@ -534,7 +602,7 @@ def submission_drafts(records: Iterable[dict[str, Any]] | str | Path) -> dict[st continue src = str(r.get("source", "")).replace("\\", "/") manifests[src] = _dir_of(src) - for p in (d.get("fields") or {}).get("storePaths") or []: + for p in _list((d.get("fields") or {}).get("storePaths")): store_paths.append((str(p).replace("\\", "/").rstrip("/"), manifests[src])) groups: dict[str, list[dict[str, Any]]] = {} planes: list[dict[str, Any]] = [] @@ -598,6 +666,12 @@ def _draft(recs: list[dict[str, Any]]) -> dict[str, Any]: image = {c: "" for c in BIL_IMAGE_COLUMNS} instrument = {c: "" for c in BIL_INSTRUMENT_COLUMNS} sources: list[str] = [] + stores = [r for r in by_format.get("ome-zarr", []) if _has_image(r)] + if stores: + # One store supplies the whole Image row. Filling blanks from the + # next store would pair one store's voxel size with another's + # shape when the channels of an acquisition differ in resolution. + by_format["ome-zarr"] = [min(stores, key=_reference_rank)] for fmt in _PREFERENCE: for r in by_format.get(fmt, []): f = (r.get("decoded") or {}).get("fields") or {} @@ -618,10 +692,48 @@ def _draft(recs: list[dict[str, Any]]) -> dict[str, Any]: # a count of files is a lower bound on slices x channels x tiles, # reported as files rather than guessed into slices. image["files"] = str(planes) - acquisition = _acquisition(by_format.get("ome-zarr", []), by_format.get("voxel-manifest", []), image, sources) + all_stores = [r for r in recs if str((r.get("decoded") or {}).get("format", "")) == "ome-zarr"] + acquisition = _acquisition(all_stores, by_format.get("voxel-manifest", []), image, sources) return {"image": image, "instrument": instrument, "sources": sources, "formats": counts, "acquisition": acquisition} +_STATE_RANK = {"complete": 0, "partial": 1, "metadata-only": 2} +_COMPARED = ("voxelSizeX", "voxelSizeY", "voxelSizeZ", "voxelSizeUnit", "sizeX", "sizeY", "sizeZ", "dtype") +_AIND_TILE_CHANNEL = re.compile(r"^(?P.+?)_ch_(?P[A-Za-z0-9]+)$") + + +def _fields(r: dict[str, Any]) -> dict[str, Any]: + f = (r.get("decoded") or {}).get("fields") + return f if isinstance(f, dict) else {} + + +def _has_image(r: dict[str, Any]) -> bool: + f = _fields(r) + return f.get("layout") != "group" and any(f.get(k) is not None for k in ("sizeX", "voxelSizeX")) + + +def _reference_rank(r: dict[str, Any]) -> tuple[int, int, float, str]: + """The store an acquisition's Image row is drafted from: a store with + a level-0 shape before one without, a complete store before a partial + one, then the finest voxel.""" + f = _fields(r) + vx = _num(f.get("voxelSizeX")) + return (0 if f.get("sizeX") is not None else 1, _STATE_RANK.get(str(f.get("state")), 3), + vx if vx is not None else float("inf"), str(r.get("source", ""))) + + +def _mismatch(stores: list[dict[str, Any]]) -> dict[str, list[Any]]: + """Fields the stores of one acquisition disagree on, with the distinct + values; a store that lacks a field does not count as disagreeing.""" + out: dict[str, list[Any]] = {} + for k in _COMPARED: + vals = {json.dumps(_fields(r)[k]) for r in stores if _fields(r).get(k) is not None} + if len(vals) > 1: + out[k] = sorted((json.loads(v) for v in vals), + key=lambda v: (not isinstance(v, (int, float)), v if isinstance(v, (int, float)) else 0, str(v))) + return out + + _MANIFEST_KEYS = (("status", "status"), ("revision", "revision"), ("startedAt", "startedAt"), ("endedAt", "endedAt"), ("volumesPlanned", "volumeCount"), ("volumesCompleted", "completedVolumes")) @@ -634,7 +746,18 @@ def _acquisition(stores: list[dict[str, Any]], manifests: list[dict[str, Any]], out: dict[str, Any] = {} if not stores and not manifests: return out - out["stores"] = len(stores) + images = [r for r in stores if _fields(r).get("layout") != "group"] + out["stores"] = len(images) + states: dict[str, int] = {} + for r in images: + st = str(_fields(r).get("state") or "unknown") + states[st] = states.get(st, 0) + 1 + if states: + out["storeStates"] = dict(sorted(states.items())) + diff = _mismatch(images) + if diff: + out["mismatch"] = diff + stores = images first_store_of: dict[str, str] = {} volumes: set[str] = set() for r in stores: @@ -643,6 +766,12 @@ def _acquisition(stores: list[dict[str, Any]], manifests: list[dict[str, Any]], view = view if isinstance(view, dict) else {} vol = _voxel_volume(store) if not view and vol is None: + # AIND's file standard names each store tile_..._ch_; + # count tiles and channels from that, and nothing else. + aind = _AIND_TILE_CHANNEL.match(_store_name(store)) + if aind: + volumes.add(aind.group("tile")) + first_store_of.setdefault(aind.group("ch"), store) continue volumes.add(str(view.get("volumeIndex")) if view.get("volumeIndex") is not None else (vol[1] if vol else store)) first_store_of.setdefault(_text(view.get("channelId")) or _store_name(store), store) @@ -656,12 +785,12 @@ def _acquisition(stores: list[dict[str, Any]], manifests: list[dict[str, Any]], for k_out, k_in in _MANIFEST_KEYS: if f.get(k_in) is not None: out[k_out] = f[k_in] - if "channels" not in out and f.get("channelIds"): - out["channels"] = [str(c) for c in f["channelIds"]] + if "channels" not in out and _list(f.get("channelIds")): + out["channels"] = [str(c) for c in _list(f.get("channelIds"))] if volumes: image["channels"] = str(len(first_store_of)) for store in first_store_of.values(): - if store not in sources: + if not any(_store_of(s) == store for s in sources): sources.append(store) elif out.get("channels") and not image["channels"]: image["channels"] = str(len(out["channels"])) diff --git a/tests/test_edge.py b/tests/test_edge.py index f183e11..bf7b71f 100644 --- a/tests/test_edge.py +++ b/tests/test_edge.py @@ -8,6 +8,7 @@ import base64 import json +from typing import Any from pathlib import Path import scigantic_bil as bil @@ -82,7 +83,8 @@ def test_prior_records_without_image_rows_still_carry_the_instrument() -> None: def test_submission_draft_from_smartspim_records(tmp_path: Path) -> None: index = tmp_path / "index.jsonl" - index.write_text("\n".join(json.dumps(r) for r in [PLANE, PLANE, SMARTSPIM, PLANE]) + '\n{"torn', encoding="utf-8") + planes = [dict(PLANE, source=PLANE["source"].replace("000000", f"{i:06d}")) for i in range(3)] + index.write_text("\n".join(json.dumps(r) for r in [planes[0], planes[1], SMARTSPIM, planes[2]]) + '\n{"torn', encoding="utf-8") draft = submission_draft(tmp_path) assert set(draft["image"]) == set(BIL_IMAGE_COLUMNS) assert set(draft["instrument"]) == set(BIL_INSTRUMENT_COLUMNS) @@ -249,14 +251,14 @@ def test_voxel_stores_group_under_their_manifest_by_each_rule() -> None: assert set(only_stores) == {root} d = only_stores[root] assert d["image"]["channels"] == "2" and d["image"]["slices"] == "10" - assert d["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"]} + assert d["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"], "storeStates": {"complete": 4}} assert d["instrument"]["microscopemanufacturerandmodel"] == "" # Without the per-store manifest view either, the .ome.zarr names and volume # directories carry the same counts. bare = json.loads(json.dumps(no_sidecar)) for r in bare: r["decoded"]["fields"].pop("manifest", None) - assert submission_draft(bare)["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"]} + assert submission_draft(bare)["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"], "storeStates": {"complete": 4}} # A voxel run under a different root is a different draft, not a blend. other = json.loads(json.dumps(no_sidecar[:1])) other[0]["source"] = other[0]["source"].replace("/run1/volumes/", "/run2/volumes/") @@ -290,7 +292,8 @@ def test_public_exaspim_stores_map_without_a_manifest() -> None: assert (img["channels"], img["slices"], img["ysize"], img["xsize"]) == ("1", "2560", "1330", "1774") assert img["dimensionorder"] == "TCZYX" assert all(v == "" for v in v2["instrument"].values()) - assert v2["acquisition"] == {"stores": 1} and v2["formats"] == {"ome-zarr": 1} + assert v2["acquisition"] == {"stores": 1, "volumes": 1, "channels": ["488"], "storeStates": {"unknown": 1}} + assert v2["formats"] == {"ome-zarr": 1} v3 = drafts["exaSPIM_2026-08-04_10-27-41/SPIM"] assert (v3["image"]["stepsizex"], v3["image"]["stepsizez"], v3["image"]["slices"]) == ("15.04", "20", "1024") assert v3["sources"] == ["exaSPIM_2026-08-04_10-27-41/SPIM/tile_000000_ch_488.ome.zarr/zarr.json"] @@ -314,3 +317,87 @@ def test_ome_zarr_voxel_size_units_reach_micrometres() -> None: def test_draft_without_stores_has_an_empty_acquisition_block() -> None: assert submission_draft([SMARTSPIM])["acquisition"] == {} + + +# --- found by stress-testing the bridge against the edge binary's own output --- + +def _store(source: str, **fields: Any) -> dict[str, Any]: + base = {"layout": "multiscales", "voxelSizeUnit": "micrometer", "state": "complete", "axes": [ + {"name": n} for n in "czyx"]} + base.update(fields) + return {"source": source, "sizeBytes": 1, "mtime": 0, "decoded": {"format": "ome-zarr", "summary": "", "fields": base}} + + +def test_watch_mode_rerecords_keep_only_the_latest_line(tmp_path: Path) -> None: + """index.jsonl is an append log; a store re-recorded as its shards land + must count once and draft from its last state, not its first.""" + early = _store("acq/volumes/000001/p/gfp.ome.zarr", state="partial", sizeZ=4, sizeY=8, sizeX=8, voxelSizeX=1, voxelSizeY=1, voxelSizeZ=2) + late = _store("acq/volumes/000001/p/gfp.ome.zarr", state="complete", sizeZ=40, sizeY=8, sizeX=8, voxelSizeX=1, voxelSizeY=1, voxelSizeZ=2) + index = tmp_path / "index.jsonl" + index.write_text("\n".join(json.dumps(r) for r in [early, early, late]) + "\n", encoding="utf-8") + d = submission_draft(index) + assert d["acquisition"]["stores"] == 1 and d["acquisition"]["storeStates"] == {"complete": 1} + assert d["image"]["slices"] == "40" + assert submission_draft([early, late])["image"]["slices"] == "40" + + +def test_container_group_joins_the_draft_of_its_tiles() -> None: + """AIND's SPIM.ome.zarr is recorded as a group and entered; it must not + split the acquisition into an empty draft of its own.""" + group = _store("ds/SPIM.ome.zarr", layout="group", state="no-array", axes=[]) + tile = _store("ds/SPIM.ome.zarr/tile_000000_ch_488.zarr", sizeZ=10, sizeY=20, sizeX=30, voxelSizeX=1, voxelSizeY=1, voxelSizeZ=1) + drafts = submission_drafts([group, tile]) + assert list(drafts) == ["ds/SPIM.ome.zarr"] + assert drafts["ds/SPIM.ome.zarr"]["acquisition"]["stores"] == 1 + + +def test_channels_at_different_resolutions_are_not_blended() -> None: + """The 2026-08-04 public exaSPIM acquisition stores 561 nm at half the + 488 nm resolution; the Image row must describe one real store and the + disagreement must be reported, not averaged into a row that is neither.""" + fine = _store("a/SPIM/tile_000000_ch_488.ome.zarr", sizeZ=1024, sizeY=2660, sizeX=3548, voxelSizeX=15.04, voxelSizeY=15.04, voxelSizeZ=20) + coarse = _store("a/SPIM/tile_000000_ch_561.ome.zarr", sizeZ=512, sizeY=1330, sizeX=1774, voxelSizeX=30.08, voxelSizeY=30.08, voxelSizeZ=40) + shapeless = _store("a/SPIM/tile_000001_ch_561.ome.zarr", state="no-array", voxelSizeX=30.08, voxelSizeY=30.08, voxelSizeZ=40) + for order in ([coarse, shapeless, fine], [shapeless, fine, coarse]): + d = submission_draft(order) + img = d["image"] + assert (img["stepsizex"], img["stepsizez"], img["xsize"], img["slices"]) == ("15.04", "20", "3548", "1024") + assert d["acquisition"]["mismatch"]["voxelSizeX"] == [15.04, 30.08] + assert d["acquisition"]["mismatch"]["sizeZ"] == [512, 1024] + assert img["channels"] == "2" and d["acquisition"]["channels"] == ["488", "561"] + assert d["acquisition"]["volumes"] == 2 + + +def test_aind_tile_names_count_tiles_and_channels() -> None: + stores = [_store(f"r/SPIM.ome.zarr/tile_x_000{x}_y_0000_z_0000_ch_{ch}.zarr", sizeZ=5, sizeY=5, sizeX=5, + voxelSizeX=0.748, voxelSizeY=0.748, voxelSizeZ=1) for x in range(3) for ch in ("488", "638")] + d = submission_draft(stores) + assert d["acquisition"]["volumes"] == 3 and d["acquisition"]["channels"] == ["488", "638"] + assert d["image"]["channels"] == "2" + + +def test_ome_zarr_scale_without_a_unit_is_not_called_micrometres() -> None: + d = submission_draft([_store("x/a.zarr", voxelSizeUnit=None, voxelSizeX=0.5, voxelSizeY=0.5, voxelSizeZ=2, sizeX=4, sizeY=4, sizeZ=4)]) + assert d["image"]["stepsizex"] == "" and d["image"]["xsize"] == "4" + d = submission_draft([_store("x/a.zarr", voxelSizeUnit="pixel", voxelSizeX=0.5, voxelSizeY=0.5, voxelSizeZ=2, sizeX=4, sizeY=4, sizeZ=4)]) + assert d["image"]["stepsizex"] == "" + d = submission_draft([_store("x/a.zarr", voxelSizeUnit="nanometer", voxelSizeX=500, voxelSizeY=500, voxelSizeZ=2000, sizeX=4, sizeY=4, sizeZ=4)]) + assert (d["image"]["stepsizex"], d["image"]["stepsizez"]) == ("0.5", "2") + + +def test_malformed_records_are_skipped_or_read_as_blank() -> None: + """A torn or hand-edited index must never crash a draft: fields that + are not objects, lists that are scalars, NaN, infinity and huge ints.""" + bad: list[Any] = [ + "not a record", 7, None, {"source": "a/x.zarr"}, {"source": "a/y.zarr", "decoded": []}, + {"source": "a/z.zarr", "decoded": {"format": "ome-zarr", "fields": [1, 2]}}, + {"source": 5, "decoded": {"format": None, "fields": None}}, + {"source": "a/manifest.json", "decoded": {"format": "voxel-manifest", "fields": { + "storePaths": True, "channelIds": 3, "laserWavelengthsNm": 488, "volumeCount": float("inf")}}}, + _store("a/volumes/000001/p/g.ome.zarr", sizeX=float("inf"), sizeY=float("nan"), sizeZ=10**40, + voxelSizeX="NaN", voxelSizeY=None, voxelSizeZ=[], axes=5, sidecars=1, manifest=[]), + ] + drafts = submission_drafts(bad) + json.dumps(drafts) + img = drafts["a"]["image"] + assert img["xsize"] == "" and img["ysize"] == "" and img["stepsizex"] == ""