diff --git a/README.md b/README.md index 173b385..4717089 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,8 @@ $ scigantic-bil submission-draft ./ctx # one draft per acquisition di The prior is the same base64 capsule the edge's own `-export-prior` writes; its card holds one `bil-image` context record per Image row of the BIL record, with the Instrument sheet folded in. The draft uses BIL's own column names, so a row pastes into the submission spreadsheet. Measured on 2026-10-17: of 1,004 light-sheet records on BIL, 849 have an Instrument block and about half of those name an objective, detector or wavelength; 808 are LifeCanvas SmartSPIM, whose `metadata.txt` carries every one of those values. +OME-Zarr stores are drafted from the edge's `ome-zarr` records: step sizes from the level-0 scale transform (converted to micrometres from whatever unit the axes name), channels, slices and array size from the shape, `dimensionorder` from the axes names (`CZYX`, `TCZYX`). An acquisition written by the Allen Institute's `voxel` engine is one store per volume and channel under `/volumes/NNNNNN//.ome.zarr` beside a `manifest.json`; those stores collapse into one draft per manifest (matched through the store's sidecar, the manifest's store paths, or the `/volumes/` prefix), with `channels` counted as the distinct `.ome.zarr` names across volumes. The manifest's `voxel-manifest` record fills the Instrument sheet (instrument name, objective, laser lines, emission bands, camera) and the draft's `acquisition` block carries what the sheets have no column for: store and volume counts, channel names, status, revision and timestamps. The microscope type is set to light-sheet only when the instrument's name says so; voxel's template instrument is a simulator. `fixtures/edge/voxel_run1_index.jsonl` is the edge decoder's own `index.jsonl` over one such run (2 tiles x 2 channels, 10 frames of 128 x 192 uint16), `exaspim_public_index.jsonl` holds hand-written records for two public exaSPIM tiles, and `tests/test_edge.py` reads both. + ## Caching On by default. Metadata responses, directory listings and manifests are cached to `~/.cache/scigantic-bil` (macOS: `~/Library/Caches/scigantic-bil`; override with `SCIGANTIC_BIL_CACHE` or `enable_cache(cache_dir=...)`) and expire after 7 days, since BIL republishes its inventory every few days. Inventory snapshots are immutable once published and are kept as plain TSV files without expiry. Image bytes are never cached. diff --git a/fixtures/edge/exaspim_public_index.jsonl b/fixtures/edge/exaspim_public_index.jsonl new file mode 100644 index 0000000..b13c684 --- /dev/null +++ b/fixtures/edge/exaspim_public_index.jsonl @@ -0,0 +1,2 @@ +{"source": "exaSPIM_851474_2026-07-10_15-23-22/SPIM.ome.zarr/tile_000000_ch_488.zarr", "sizeBytes": 7759, "mtime": 1784108793, "decoded": {"format": "ome-zarr", "summary": "OME-Zarr v2 (NGFF 0.4) multiscales, 7 levels, tczyx 1x1x2560x1330x1774 uint16, 8 x 5.984 x 5.984 micrometer, blosc/zstd", "fields": {"zarrFormat": 2, "ngffVersion": "0.4", "layout": "multiscales", "axes": [{"name": "t", "type": "time", "unit": "millisecond"}, {"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "levels": 7, "datasetPaths": ["0", "1", "2", "3", "4", "5", "6"], "sizeT": 1, "sizeC": 1, "sizeZ": 2560, "sizeY": 1330, "sizeX": 1774, "dtype": "uint16", "fillValue": 0, "chunkShape": [1, 1, 128, 256, 256], "shardShape": null, "codecs": ["blosc"], "compressor": "blosc/zstd", "compressionLevel": 1, "voxelSizeX": 5.984, "voxelSizeY": 5.984, "voxelSizeZ": 8.0, "voxelSizeUnit": "micrometer", "channelNames": ["Channel:tile_000000_ch_488.zarr:0"], "omeroPresent": true, "presentChunks": 0, "presentChunksIsLowerBound": true, "metadataFiles": [".zgroup", ".zattrs", "0/.zarray", "6/.zarray"], "arrayMetadataMissing": ["1/.zarray", "2/.zarray", "3/.zarray", "4/.zarray", "5/.zarray"], "sidecars": []}}} +{"source": "exaSPIM_2026-08-04_10-27-41/SPIM/tile_000000_ch_488.ome.zarr/zarr.json", "sizeBytes": 1849, "mtime": 1789153861, "decoded": {"format": "ome-zarr", "summary": "OME-Zarr v3 (NGFF 0.5) multiscales, 6 levels, tczyx 1x1x1024x2660x3548 uint16, 20 x 15.04 x 15.04 micrometer, sharded, zstd", "fields": {"zarrFormat": 3, "ngffVersion": "0.5", "layout": "multiscales", "axes": [{"name": "t", "type": "time", "unit": "millisecond"}, {"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "levels": 6, "datasetPaths": ["0", "1", "2", "3", "4", "5"], "sizeT": 1, "sizeC": 1, "sizeZ": 1024, "sizeY": 2660, "sizeX": 3548, "dtype": "uint16", "fillValue": 0, "chunkShape": [1, 1, 128, 256, 256], "shardShape": [1, 1, 512, 512, 512], "codecs": ["sharding_indexed", "transpose", "bytes", "zstd"], "compressor": "zstd", "compressionLevel": 3, "voxelSizeX": 15.039999977980552, "voxelSizeY": 15.040000146851504, "voxelSizeZ": 20.0, "voxelSizeUnit": "micrometer", "channelNames": ["tile_000000_ch_488"], "omeroPresent": true, "presentChunks": 0, "presentChunksIsLowerBound": true, "metadataFiles": ["zarr.json", "0/zarr.json", "1/zarr.json", "2/zarr.json", "3/zarr.json", "4/zarr.json", "5/zarr.json"], "arrayMetadataMissing": [], "sidecars": []}}} diff --git a/fixtures/edge/voxel_run1_index.jsonl b/fixtures/edge/voxel_run1_index.jsonl new file mode 100644 index 0000000..63f5f74 --- /dev/null +++ b/fixtures/edge/voxel_run1_index.jsonl @@ -0,0 +1,5 @@ +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/manifest.json","sizeBytes":12515,"mtime":1792497656,"decoded":{"format":"voxel-manifest","summary":"voxel acquisition sim, completed, 4 volume(s), 4/4 dataset(s) completed, 2 channel(s) gfp/rfp, 488.0/561.0 nm, 8.0x","fields":{"availableLocations":4,"cameraDriver":"vxl.devices.camera.simulated.SimulatedCamera","cameraIds":["camera_1"],"channelEmissionNm":{"gfp":510,"rfp":600},"channelExcitationNm":{"gfp":488,"rfp":561},"channelIds":["gfp","rfp"],"channelLabels":{"gfp":"GFP","rfp":"RFP"},"completedDatasets":4,"completedVolumes":4,"createdAt":"2026-10-20T12:00:29.321395Z","datasetCount":4,"datasetStatusCounts":{"completed":4},"endedAt":"2026-10-20T12:00:56.503012Z","failure":null,"host":"Aarons-MacBook-Air.local","id":"26fdfb5c-31dd-40e6-8e8f-4f397fb53026","instrumentName":"sim","laserWavelengthsNm":[488,561],"objectiveMagnification":8,"operator":"scratch","output":{"batchZShards":1,"compression":"blosc.lz4","downscaleType":"gaussian","maxLevel":3,"shardZChunks":1,"targetShardGb":1},"planPoints":2,"profiles":["single_gfp","single_rfp"],"remoteRoot":null,"revision":15,"schemaVersion":"1.0","sensorHeightPx":128,"sensorPixelSizeUm":3.76,"sensorWidthPx":192,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","storePaths":["volumes/000001/single_gfp/gfp.ome.zarr","volumes/000002/single_rfp/rfp.ome.zarr","volumes/000003/single_gfp/gfp.ome.zarr","volumes/000004/single_rfp/rfp.ome.zarr"],"subjectId":null,"volumeCount":4,"volumeStatusCounts":{"completed":4},"volumes":[{"channel":"gfp","datasetStatus":"completed","format":"ome-zarr","index":1,"locationStatus":"available","profile":"single_gfp","storePath":"volumes/000001/single_gfp/gfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},{"channel":"rfp","datasetStatus":"completed","format":"ome-zarr","index":2,"locationStatus":"available","profile":"single_rfp","storePath":"volumes/000002/single_rfp/rfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},{"channel":"gfp","datasetStatus":"completed","format":"ome-zarr","index":3,"locationStatus":"available","profile":"single_gfp","storePath":"volumes/000003/single_gfp/gfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1},{"channel":"rfp","datasetStatus":"completed","format":"ome-zarr","index":4,"locationStatus":"available","profile":"single_rfp","storePath":"volumes/000004/single_rfp/rfp.ome.zarr","task":"tiles","volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1}]}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000001/single_gfp/gfp.ome.zarr","sizeBytes":5545,"mtime":1792497636,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"gfp","channelLabel":"GFP","datasetStatus":"completed","emissionNm":510,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":488,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_gfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":1,"volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.29692,"endedAt":"2026-10-20T12:00:36.952690Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:32.655770Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000001/single_gfp/gfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000002/single_rfp/rfp.ome.zarr","sizeBytes":5545,"mtime":1792497643,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"rfp","channelLabel":"RFP","datasetStatus":"completed","emissionNm":600,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":561,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_rfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":2,"volumeStatus":"completed","x":0,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.215291,"endedAt":"2026-10-20T12:00:43.308542Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:39.093251Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000002/single_rfp/rfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000003/single_gfp/gfp.ome.zarr","sizeBytes":5545,"mtime":1792497650,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"gfp","channelLabel":"GFP","datasetStatus":"completed","emissionNm":510,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":488,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_gfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":3,"volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.18648,"endedAt":"2026-10-20T12:00:50.197589Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:46.011109Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000003/single_gfp/gfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} +{"source":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000004/single_rfp/rfp.ome.zarr","sizeBytes":5545,"mtime":1792497656,"decoded":{"format":"ome-zarr","summary":"OME-Zarr 0.5 (Zarr v3), 1x10x128x192 czyx uint16, 0.47x0.47x1 micrometer, 1 channel(s), 4 level(s), shards 1x64x128x192 of chunks 1x64x64x64, blosc/lz4 3, 1/1 shard(s), voxel dataset completed, 2 sidecar(s)","fields":{"acquisition":null,"arrayMetadataMissing":false,"axes":[{"name":"c","type":"channel","unit":null},{"name":"z","type":"space","unit":"micrometer"},{"name":"y","type":"space","unit":"micrometer"},{"name":"x","type":"space","unit":"micrometer"}],"bdvReferencesStore":null,"bioformats2rawLayout":null,"channelColors":null,"channelNames":[],"channels":1,"chunkKeySeparator":"/","chunkShape":[1,64,64,64],"codecs":["bytes","blosc"],"completeness":1,"compressionLevel":3,"compressor":"blosc/lz4","compressorShuffle":"bitshuffle","creator":null,"datasetPaths":["0","1","2","3"],"dimensionNames":["c","z","y","x"],"dtype":"uint16","dtypeRaw":"uint16","expectedChunks":1,"fillValue":0,"imagePath":"","innerChunksExpected":6,"layout":"multiscales","levelExpectedChunks":[1,1,1,1],"levelShapes":[[1,10,128,192],[1,5,64,96],[1,2,32,48],[1,1,16,24]],"levels":4,"levelsWithArrayMetadata":4,"manifest":{"channelId":"rfp","channelLabel":"RFP","datasetStatus":"completed","emissionNm":600,"endedAt":"2026-10-20T12:00:56.503012Z","excitationWavelengthNm":561,"instrumentName":"sim","locationStatus":"available","objectiveMagnification":8,"profile":"single_rfp","revision":15,"startedAt":"2026-10-20T12:00:29.325932Z","status":"completed","storagePath":"run1","volumeIndex":4,"volumeStatus":"completed","x":500,"y":0,"zEnd":9,"zStart":0,"zStep":1},"members":null,"metadataBytesRead":3321,"metadataFiles":["0/zarr.json","1/zarr.json","2/zarr.json","3/zarr.json","zarr.json"],"metadataFilesMissing":[],"metadataFilesUnreadable":[],"metrics":{"batchZ":8,"batchZShards":1,"batches":2,"channelIndex":0,"collectedFrames":10,"compression":"blosc.lz4","downscaleType":"gaussian","dtype":"uint16","durationS":4.162044,"endedAt":"2026-10-20T12:00:56.457014Z","errors":0,"expectedFrames":10,"flushedBytes":559872,"maxLevel":3,"schemaVersion":1,"shardZChunks":1,"slots":4,"startedAt":"2026-10-20T12:00:52.294970Z","target":"/private/tmp/claude-501/-Users-aaronkanzer-projects-scigantic/38e183f3-8f0d-4740-9830-0c8b76011490/scratchpad/voxel/out/run1/store/run1/volumes/000004/single_rfp/rfp.ome.zarr","targetShardGb":1,"transferedBytes":0,"volumeShape":"10,128,192","volumeShapeZYX":[10,128,192],"voxelSize":"1.0,0.47,0.47","voxelSizeZYX":[1,0.47,0.47],"voxelUnit":"micrometer"},"metricsPresent":true,"multiscaleName":null,"multiscaleType":"gaussian","ngffVersion":"0.5","omeroPresent":false,"plate":null,"presentChunks":1,"presentChunksIsLowerBound":false,"scale":[1,1,0.47,0.47],"series":null,"shape":[1,10,128,192],"shardShape":[1,64,128,192],"sharded":true,"sidecarFormats":["voxel-metrics","voxel-manifest"],"sidecars":["metrics.json","../../../../manifest.json"],"sizeC":1,"sizeX":192,"sizeY":128,"sizeZ":10,"state":"complete","tilePositionX":null,"tilePositionY":null,"tilePositionZ":null,"timeStep":null,"timeUnit":null,"translation":null,"voxelSizeUnit":"micrometer","voxelSizeX":0.47,"voxelSizeY":0.47,"voxelSizeZ":1,"well":null,"zarrFormat":3}}} diff --git a/src/scigantic_bil/edge.py b/src/scigantic_bil/edge.py index 6aac3da..493d913 100644 --- a/src/scigantic_bil/edge.py +++ b/src/scigantic_bil/edge.py @@ -16,8 +16,9 @@ box wrote while watching the acquisition and fills in BIL's Image and Instrument submission sheets from the headers it decoded (SmartSPIM metadata.txt, Zeiss CZI, Luxendo and mesoSPIM sidecars, lattice Settings.txt, -BigDataViewer XML, Micro-Manager metadata, OME-TIFF, Imaris, KLB, ND2). On -2026-10-17, 849 of 1,004 light-sheet records on BIL had an Instrument +BigDataViewer XML, Micro-Manager metadata, OME-TIFF, Imaris, KLB, ND2, +OME-Zarr stores and the ``manifest.json`` of an Allen Institute ``voxel`` +acquisition). On 2026-10-17, 849 of 1,004 light-sheet records on BIL had an Instrument block and about half of those had objective, detector and wavelength fields; the rest were left blank by hand. These are exactly the values the microscope wrote beside its planes. @@ -26,12 +27,22 @@ (``stepsizex``, ``microscopemanufacturerandmodel`` and so on), so a draft row can be pasted into the submission spreadsheet column for column. + +The prior's ``bil-image`` fields already cover what an OME-Zarr record on +the box can say about itself: ``stepSizeXUm``/``YUm``/``ZUm`` against the +store's level-0 scale in micrometres, ``channels`` against ``sizeC``, +``slices`` and ``sizeZ`` against ``sizeZ``, ``sizeX``/``sizeY`` against the +array shape, ``dimensionOrder`` against the axes names. Nothing OME-Zarr +specific is added to the prior. """ from __future__ import annotations import base64 import json +import math +import posixpath +import re from pathlib import Path from typing import Any, Iterable, Iterator @@ -69,7 +80,7 @@ def _num(value: Any) -> float | None: v = float(str(value).strip()) except ValueError: return None - return v if v == v else None # NaN check + return v if math.isfinite(v) else None # NaN and +-inf are not values def _int(value: Any) -> int | None: @@ -77,6 +88,11 @@ def _int(value: Any) -> int | None: return int(v) if v is not None and v > 0 else None +def _list(value: Any) -> list[Any]: + """``value`` if the decoder wrote a list there, else an empty list.""" + return value if isinstance(value, list) else [] + + def _text(value: Any) -> str | None: s = str(value).strip() if value is not None else "" return s or None @@ -171,10 +187,18 @@ def write_prior(detail: DatasetDetail | str, path: str | Path) -> Path: def read_index(path: str | Path) -> Iterator[dict[str, Any]]: """Yield the records of an edge box's ``index.jsonl`` (or a directory - holding one), tolerating a torn last line.""" + holding one), tolerating a torn last line. + + ``index.jsonl`` is an append log: a box in ``-watch`` mode writes a new + line each time a source changes (an OME-Zarr store re-records as its + shards land, a voxel ``manifest.json`` is rewritten on every status + change). Only the last line for each source is yielded, in the order + sources first appeared, which is what the box's own resume reads too. + """ p = Path(path) if p.is_dir(): p = p / "index.jsonl" + recs: list[dict[str, Any]] = [] with p.open("r", encoding="utf-8") as fh: for line in fh: line = line.strip() @@ -185,16 +209,80 @@ def read_index(path: str | Path) -> Iterator[dict[str, Any]]: except json.JSONDecodeError: continue if isinstance(rec, dict) and isinstance(rec.get("decoded"), dict): - yield rec + recs.append(rec) + yield from _latest(recs) + + +def _clean(r: Any) -> dict[str, Any] | None: + """A record with a dict ``decoded``, a dict ``fields`` and string + ``format`` and ``source``, or ``None`` for one that is not a record. + Every reader below can then index into it without checking types.""" + if not isinstance(r, dict) or not isinstance(r.get("decoded"), dict): + return None + d = dict(r["decoded"]) + d["format"] = str(d.get("format") or "") + d["fields"] = d["fields"] if isinstance(d.get("fields"), dict) else {} + src = r.get("source") + return dict(r, decoded=d, source=src if isinstance(src, str) else "") + + +def _latest(records: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + """The last record for each ``source``, in first-seen order. Records + with no source are kept as they are.""" + by_source: dict[str, dict[str, Any]] = {} + loose: list[tuple[int, dict[str, Any]]] = [] + order: dict[str, int] = {} + for i, raw in enumerate(records): + r = _clean(raw) + if r is None: + continue + src = str(r.get("source", "")).replace("\\", "/").rstrip("/") + if not src: + loose.append((i, r)) + continue + order.setdefault(src, i) + by_source[src] = r + keyed = [(order[k], v) for k, v in by_source.items()] + loose + return [r for _, r in sorted(keyed, key=lambda t: t[0])] + + +# Micrometres per unit, keyed by the spellings seen in BigDataViewer XML, +# OME-TIFF (symbols) and OME-NGFF axes (UDUNITS-2 names: "micrometer", +# "nanometer", "millimeter"). British spellings and plurals are folded in +# before lookup. An unknown or missing unit is taken as micrometres, the +# convention every other decoder here already relies on. +_UM_PER_UNIT = { + "pm": 1e-6, "picometer": 1e-6, + "nm": 1e-3, "nanometer": 1e-3, + "um": 1.0, "\u00b5m": 1.0, "\u03bcm": 1.0, "micrometer": 1.0, "micron": 1.0, + "mm": 1e3, "millimeter": 1e3, + "cm": 1e4, "centimeter": 1e4, + "m": 1e6, "meter": 1e6, + "angstrom": 1e-4, "\u00e5": 1e-4, "\u212b": 1e-4, +} def _um(value: Any, unit: Any = None) -> float | None: v = _num(value) if v is None: return None - u = (str(unit) if unit else "").strip().lower() - scale = {"nm": 1e-3, "mm": 1e3, "m": 1e6, "cm": 1e4, "pm": 1e-6}.get(u, 1.0) - return v * scale + u = (str(unit) if unit else "").strip().lower().replace("metre", "meter") + if u not in _UM_PER_UNIT and len(u) > 3 and u.endswith("s"): + u = u[:-1] # microns, micrometers; never "ms" + return v * _UM_PER_UNIT.get(u, 1.0) + + +def _um_declared(value: Any, unit: Any) -> float | None: + """Like :func:`_um`, but ``None`` when the unit is missing or not a + length. OME-NGFF makes an axis unit optional, and a scale with no unit + is in no known unit at all, so it is not reported as micrometres.""" + v = _num(value) + u = (str(unit) if unit else "").strip().lower().replace("metre", "meter") + if u not in _UM_PER_UNIT and len(u) > 3 and u.endswith("s"): + u = u[:-1] + if v is None or u not in _UM_PER_UNIT: + return None + return v * _UM_PER_UNIT[u] def _fmt(v: float | None) -> str: @@ -249,7 +337,7 @@ def _image_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: sx, sy, sz = _num(f.get("voxelSizeXUm")), _num(f.get("voxelSizeYUm")), _num(f.get("voxelSizeZUm")) slices = _int(f.get("sizeZ")) xsize, ysize, zsize = _int(f.get("sizeX")), _int(f.get("sizeY")), _int(f.get("sizeZ")) - on = f.get("lasersOnNm") or [] + on = _list(f.get("lasersOnNm")) channels = len(on) if on else None elif fmt == "lls-settings": sz = _num(f.get("stepUm")) @@ -263,6 +351,16 @@ def _image_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: elif fmt == "nd2": sx = sy = _num(f.get("pixelSizeUm")) xsize, ysize = _int(f.get("imageWidth")), _int(f.get("imageLength")) + elif fmt == "ome-zarr": + # voxelSize* is the level-0 scale transform in voxelSizeUnit (an + # NGFF axis unit name, "micrometer" in every store seen so far). + # A missing or non-length unit leaves the step sizes blank. + u = f.get("voxelSizeUnit") + sx, sy, sz = (_um_declared(f.get(k), u) for k in ("voxelSizeX", "voxelSizeY", "voxelSizeZ")) + channels, slices = _int(f.get("sizeC")), _int(f.get("sizeZ")) + xsize, ysize, zsize = _int(f.get("sizeX")), _int(f.get("sizeY")), _int(f.get("sizeZ")) + names = [str(a.get("name") or "") for a in _list(f.get("axes")) if isinstance(a, dict)] + order = "".join(names).upper() or None else: return row row["stepsizex"], row["stepsizey"], row["stepsizez"] = _fmt(sx), _fmt(sy), _fmt(sz) @@ -281,7 +379,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivename"] = _text(f.get("objective")) or "" row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["objectiveimmersion"] = _text(f.get("immersion")) or "" - lines = [str(int(_num(w) or 0)) for w in (f.get("channelWavelengthsNm") or []) if _num(w)] + lines = [str(int(_num(w) or 0)) for w in _list(f.get("channelWavelengthsNm")) if _num(w)] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" elif fmt == "czi": @@ -293,11 +391,11 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("objectiveNA"))) row["objectivemagnification"] = _fmt(_num(f.get("objectiveMagnification"))) row["objectiveimmersion"] = _text(f.get("objectiveImmersion")) or "" - dets = [d for d in (f.get("detectorModels") or []) if d] + dets = [d for d in _list(f.get("detectorModels")) if d] row["detectormodel"] = ", ".join(str(d) for d in dets) row["detectortype"] = "CMOS" if dets else "" - ex = [_fmt(_num(v)) for v in (f.get("channelExcitationNm") or []) if _num(v)] - em = [_fmt(_num(v)) for v in (f.get("channelEmissionNm") or []) if _num(v)] + ex = [_fmt(_num(v)) for v in _list(f.get("channelExcitationNm")) if _num(v)] + em = [_fmt(_num(v)) for v in _list(f.get("channelEmissionNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(ex) + (" nm" if ex else "") row["detectionwavelength"] = ", ".join(em) + (" nm" if em else "") elif fmt == "imaris": @@ -305,7 +403,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["microscopetype"] = "Light-sheet" if "lightsheet" in mode.lower().replace(" ", "") else mode row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["objectivena"] = _fmt(_num(f.get("numericalAperture"))) - ex = [_fmt(_num(v)) for v in (f.get("excitationWavelengthsNm") or []) if _num(v)] + ex = [_fmt(_num(v)) for v in _list(f.get("excitationWavelengthsNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(ex) + (" nm" if ex else "") elif fmt == "bdv-spimdata": row["microscopetype"] = "Light-sheet" @@ -316,7 +414,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["detectortype"] = "CMOS" if f.get("cameraType") else "" row["detectormodel"] = _text(f.get("cameraType")) or "" - lines = [str(int(_num(str(w).split()[0]) or 0)) for w in (f.get("lasers") or []) if _num(str(w).split()[0])] + lines = [str(int(_num(str(w).split()[0]) or 0)) for w in _list(f.get("lasers")) if _num(str(w).split()[0])] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" elif fmt == "luxendo-stack": @@ -327,7 +425,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("numericalAperture"))) row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["objectiveimmersion"] = _text(f.get("objectiveImmersion")) or "" - on = [_fmt(_num(v)) for v in (f.get("lasersOnNm") or []) if _num(v)] + on = [_fmt(_num(v)) for v in _list(f.get("lasersOnNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(on) + (" nm" if on else "") row["illuminationtypes"] = "laser" if on else "" elif fmt == "lls-settings": @@ -335,7 +433,7 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["microscopemanufacturerandmodel"] = "Lattice light-sheet (Janelia design)" row["detectortype"] = "CMOS" if f.get("cameraModel") else "" row["detectormodel"] = _text(f.get("cameraModel")) or "" - lines = [_fmt(_num(v)) for v in (f.get("laserLinesNm") or []) if _num(v)] + lines = [_fmt(_num(v)) for v in _list(f.get("laserLinesNm")) if _num(v)] row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") row["illuminationtypes"] = "laser" if lines else "" elif fmt == "nd2": @@ -344,13 +442,36 @@ def _instrument_row_from(fmt: str, f: dict[str, Any]) -> dict[str, str]: row["objectivena"] = _fmt(_num(f.get("numericalAperture"))) row["objectivemagnification"] = _fmt(_num(f.get("magnification"))) row["detectormodel"] = _text(f.get("cameraName")) or "" + elif fmt == "voxel-manifest": + # voxel is a generic acquisition engine; its template instrument is + # a simulator. The manifest says light-sheet only when the + # instrument's name does (exaSPIM, diSPIM, ...), so the type is left + # blank otherwise rather than guessed from the engine's origin. + name = _text(f.get("instrumentName")) or "" + row["microscopetype"] = "Light-sheet" if _names_light_sheet(name) else "" + row["microscopemanufacturerandmodel"] = name + row["objectivename"] = _text(f.get("objectiveName")) or "" + row["objectivena"] = _fmt(_num(f.get("objectiveNA"))) + row["objectivemagnification"] = _fmt(_num(f.get("objectiveMagnification"))) + row["detectormodel"] = _text(f.get("cameraModel")) or "" + lines = [_fmt(_num(v)) for v in _list(f.get("laserWavelengthsNm")) if _num(v)] + row["illuminationwavelength"] = ", ".join(lines) + (" nm" if lines else "") + row["illuminationtypes"] = "laser" if lines else "" + em_table = f.get("channelEmissionNm") + em = [_fmt(_num(v)) for v in (em_table.values() if isinstance(em_table, dict) else []) if _num(v)] + row["detectionwavelength"] = ", ".join(em) + (" nm" if em else "") return row +def _names_light_sheet(name: str) -> bool: + flat = name.lower().replace("-", "").replace("_", "").replace(" ", "") + return any(tag in flat for tag in ("lightsheet", "spim", "lsfm")) + + # Formats ranked by how much of the submission they can fill; the best # available one supplies each column, later ones only fill blanks. -_PREFERENCE = ("smartspim-metadata", "czi", "luxendo-stack", "mesospim-meta", "lls-settings", "bdv-spimdata", - "micromanager-metadata", "ome-tiff", "imaris", "klb", "nd2") +_PREFERENCE = ("smartspim-metadata", "czi", "luxendo-stack", "mesospim-meta", "lls-settings", "voxel-manifest", + "bdv-spimdata", "micromanager-metadata", "ome-zarr", "ome-tiff", "imaris", "klb", "nd2") def _dir_of(source: str) -> str: @@ -359,6 +480,79 @@ def _dir_of(source: str) -> str: _PLANE_FORMATS = frozenset({"png", "tiff", "dcimg"}) +_STORE_METADATA = frozenset({"zarr.json", ".zattrs", ".zgroup", ".zarray"}) + + +def _store_of(source: str) -> str: + """The store directory an ``ome-zarr`` record describes: the deepest + path component ending in ``.zarr``, so a record keyed on the store's + ``zarr.json`` or ``.zattrs`` and one keyed on the directory agree. A + source with no such component is its own store (or, for a bare + metadata file, its directory).""" + s = source.replace("\\", "/").rstrip("/") + parts = s.split("/") + for i in range(len(parts), 0, -1): + if parts[i - 1].lower().endswith(".zarr"): + return "/".join(parts[:i]) + if parts[-1] in _STORE_METADATA: + return "/".join(parts[:-1]) + return s + + +def _store_name(store: str) -> str: + base = store.rsplit("/", 1)[-1] + low = base.lower() + for suffix in (".ome.zarr", ".zarr"): + if low.endswith(suffix): + return base[: -len(suffix)] + return base + + +def _voxel_volume(store: str) -> tuple[str, str] | None: + """``(acquisition root, volume directory)`` for a store laid out as + voxel writes it, ``/volumes/NNNNNN//.ome.zarr``; + ``None`` for any other path.""" + parts = store.split("/") + for i in range(len(parts) - 2): + if parts[i] == "volumes" and parts[i + 1].isdigit(): + return "/".join(parts[:i]), "/".join(parts[: i + 2]) + return None + + +def _manifest_candidates(store: str, f: dict[str, Any]) -> list[str]: + """Where a store's ``manifest.json`` sidecar would sit in the index: + the sidecar resolved against the store directory (the decoder writes + sidecars relative to the store) and the sidecar as written.""" + out: list[str] = [] + for sc in _list(f.get("sidecars")): + s = str(sc).replace("\\", "/") + if s.rsplit("/", 1)[-1] != "manifest.json": + continue + if not s.startswith("/"): + out.append(posixpath.normpath(posixpath.join(store, s))) + out.append(s) + return out + + +def _store_home(source: str, f: dict[str, Any], manifests: dict[str, str], store_paths: list[tuple[str, str]]) -> str: + """The acquisition directory an ``ome-zarr`` record belongs to; the + rule is spelled out on :func:`submission_drafts`.""" + store = _store_of(source) + if f.get("layout") == "group": + # A plain container group (AIND's SPIM.ome.zarr holding one store + # per tile and channel) is the acquisition its member stores sit + # in, so it joins their draft instead of starting one of its own. + return store + for cand in _manifest_candidates(store, f): + if cand in manifests: + return manifests[cand] + for path, home in store_paths: + if store in (path, posixpath.normpath(posixpath.join(home, path)) if home else path) or path.endswith("/" + store): + return home + vol = _voxel_volume(store) + if vol is not None: + return vol[0] + return _dir_of(store) def submission_drafts(records: Iterable[dict[str, Any]] | str | Path) -> dict[str, dict[str, Any]]: @@ -371,16 +565,57 @@ def submission_drafts(records: Iterable[dict[str, Any]] | str | Path) -> dict[st plain TIFF) are counted under the deepest such directory that contains them. Returns ``{directory: draft}`` with the same draft shape as :func:`submission_draft`. + + An ``ome-zarr`` record is keyed on its store (the deepest path + component ending in ``.zarr``, whether the record names the directory + or its ``zarr.json``) and goes to the first of these that applies: + + 1. the directory of a ``voxel-manifest`` record that one of the store's + ``sidecars`` points at (``../../../manifest.json`` resolved against + the store, or the sidecar as written); + 2. the directory of a ``voxel-manifest`` record whose ``storePaths`` + lists the store (relative to the manifest's directory, as the + decoder writes them, or equal to the store's path); + 3. the path prefix before ``/volumes/NNNNNN/``, voxel's layout, when + no manifest has been indexed yet; + 4. the store's parent directory, like any other file. The public + AIND layout ``SPIM.ome.zarr/tile_NNNNNN_ch_NNN.zarr`` therefore + collapses to one draft per ``SPIM.ome.zarr``. + + A voxel acquisition is one BIL submission made of many single-channel + stores (one per volume and channel). Within such a group the Image + row's ``channels`` is the number of distinct channels across the + volumes (each store's ``fields.manifest.channelId``, or its + ``.ome.zarr`` name when the decoder paired no manifest), not + the ``sizeC`` of one store (always 1); volumes are counted the same + way (``fields.manifest.volumeIndex``, else the ``volumes/NNNNNN`` + directory). The draft's ``acquisition`` block carries what BIL's + sheets have no column for: store and volume counts, channel names, + the manifest's status, revision and timestamps. """ - recs = list(read_index(records)) if isinstance(records, (str, Path)) else list(records) + recs = list(read_index(records)) if isinstance(records, (str, Path)) else _latest(records) + manifests: dict[str, str] = {} + store_paths: list[tuple[str, str]] = [] + for r in recs: + d = r.get("decoded") or {} + if str(d.get("format", "")) != "voxel-manifest": + continue + src = str(r.get("source", "")).replace("\\", "/") + manifests[src] = _dir_of(src) + for p in _list((d.get("fields") or {}).get("storePaths")): + store_paths.append((str(p).replace("\\", "/").rstrip("/"), manifests[src])) groups: dict[str, list[dict[str, Any]]] = {} planes: list[dict[str, Any]] = [] for r in recs: - fmt = str((r.get("decoded") or {}).get("format", "")) + d = r.get("decoded") or {} + fmt = str(d.get("format", "")) + src = str(r.get("source", "")) if fmt in _PLANE_FORMATS: planes.append(r) + elif fmt == "ome-zarr": + groups.setdefault(_store_home(src, d.get("fields") or {}, manifests, store_paths), []).append(r) else: - groups.setdefault(_dir_of(str(r.get("source", ""))), []).append(r) + groups.setdefault(_dir_of(src), []).append(r) if not groups: groups[""] = [] dirs = sorted(groups, key=len, reverse=True) @@ -402,11 +637,13 @@ def submission_draft(records: Iterable[dict[str, Any]] | str | Path, cohort: str instruments into one deposit. Returns ``{"image": {...}, "instrument": {...}, "sources": [...], - "formats": {...}}`` with every BIL column present (blank where - nothing on the box says), the records each value came from, and a - per-format count of what was read. Plane files (PNG, plain TIFF) are - counted, not used: they say width and height, which BIL does not ask - for. + "formats": {...}, "acquisition": {...}}`` with every BIL column + present (blank where nothing on the box says), the records each value + came from, a per-format count of what was read, and, for OME-Zarr + stores and voxel manifests, the counts and status BIL has no column + for (empty otherwise; see :func:`submission_drafts`). Plane files + (PNG, plain TIFF) are counted, not used: they say width and height, + which BIL does not ask for. """ drafts = submission_drafts(records) if cohort is not None: @@ -429,6 +666,12 @@ def _draft(recs: list[dict[str, Any]]) -> dict[str, Any]: image = {c: "" for c in BIL_IMAGE_COLUMNS} instrument = {c: "" for c in BIL_INSTRUMENT_COLUMNS} sources: list[str] = [] + stores = [r for r in by_format.get("ome-zarr", []) if _has_image(r)] + if stores: + # One store supplies the whole Image row. Filling blanks from the + # next store would pair one store's voxel size with another's + # shape when the channels of an acquisition differ in resolution. + by_format["ome-zarr"] = [min(stores, key=_reference_rank)] for fmt in _PREFERENCE: for r in by_format.get(fmt, []): f = (r.get("decoded") or {}).get("fields") or {} @@ -449,4 +692,106 @@ def _draft(recs: list[dict[str, Any]]) -> dict[str, Any]: # a count of files is a lower bound on slices x channels x tiles, # reported as files rather than guessed into slices. image["files"] = str(planes) - return {"image": image, "instrument": instrument, "sources": sources, "formats": counts} + all_stores = [r for r in recs if str((r.get("decoded") or {}).get("format", "")) == "ome-zarr"] + acquisition = _acquisition(all_stores, by_format.get("voxel-manifest", []), image, sources) + return {"image": image, "instrument": instrument, "sources": sources, "formats": counts, "acquisition": acquisition} + + +_STATE_RANK = {"complete": 0, "partial": 1, "metadata-only": 2} +_COMPARED = ("voxelSizeX", "voxelSizeY", "voxelSizeZ", "voxelSizeUnit", "sizeX", "sizeY", "sizeZ", "dtype") +_AIND_TILE_CHANNEL = re.compile(r"^(?P.+?)_ch_(?P[A-Za-z0-9]+)$") + + +def _fields(r: dict[str, Any]) -> dict[str, Any]: + f = (r.get("decoded") or {}).get("fields") + return f if isinstance(f, dict) else {} + + +def _has_image(r: dict[str, Any]) -> bool: + f = _fields(r) + return f.get("layout") != "group" and any(f.get(k) is not None for k in ("sizeX", "voxelSizeX")) + + +def _reference_rank(r: dict[str, Any]) -> tuple[int, int, float, str]: + """The store an acquisition's Image row is drafted from: a store with + a level-0 shape before one without, a complete store before a partial + one, then the finest voxel.""" + f = _fields(r) + vx = _num(f.get("voxelSizeX")) + return (0 if f.get("sizeX") is not None else 1, _STATE_RANK.get(str(f.get("state")), 3), + vx if vx is not None else float("inf"), str(r.get("source", ""))) + + +def _mismatch(stores: list[dict[str, Any]]) -> dict[str, list[Any]]: + """Fields the stores of one acquisition disagree on, with the distinct + values; a store that lacks a field does not count as disagreeing.""" + out: dict[str, list[Any]] = {} + for k in _COMPARED: + vals = {json.dumps(_fields(r)[k]) for r in stores if _fields(r).get(k) is not None} + if len(vals) > 1: + out[k] = sorted((json.loads(v) for v in vals), + key=lambda v: (not isinstance(v, (int, float)), v if isinstance(v, (int, float)) else 0, str(v))) + return out + + +_MANIFEST_KEYS = (("status", "status"), ("revision", "revision"), ("startedAt", "startedAt"), ("endedAt", "endedAt"), + ("volumesPlanned", "volumeCount"), ("volumesCompleted", "completedVolumes")) + + +def _acquisition(stores: list[dict[str, Any]], manifests: list[dict[str, Any]], + image: dict[str, str], sources: list[str]) -> dict[str, Any]: + """The ``acquisition`` block of a draft, and the one Image column it + corrects: ``channels`` of a voxel acquisition, counted across its + single-channel stores.""" + out: dict[str, Any] = {} + if not stores and not manifests: + return out + images = [r for r in stores if _fields(r).get("layout") != "group"] + out["stores"] = len(images) + states: dict[str, int] = {} + for r in images: + st = str(_fields(r).get("state") or "unknown") + states[st] = states.get(st, 0) + 1 + if states: + out["storeStates"] = dict(sorted(states.items())) + diff = _mismatch(images) + if diff: + out["mismatch"] = diff + stores = images + first_store_of: dict[str, str] = {} + volumes: set[str] = set() + for r in stores: + store = _store_of(str(r.get("source", ""))) + view = ((r.get("decoded") or {}).get("fields") or {}).get("manifest") + view = view if isinstance(view, dict) else {} + vol = _voxel_volume(store) + if not view and vol is None: + # AIND's file standard names each store tile_..._ch_; + # count tiles and channels from that, and nothing else. + aind = _AIND_TILE_CHANNEL.match(_store_name(store)) + if aind: + volumes.add(aind.group("tile")) + first_store_of.setdefault(aind.group("ch"), store) + continue + volumes.add(str(view.get("volumeIndex")) if view.get("volumeIndex") is not None else (vol[1] if vol else store)) + first_store_of.setdefault(_text(view.get("channelId")) or _store_name(store), store) + if volumes: + out["volumes"] = len(volumes) + out["channels"] = sorted(first_store_of) + if manifests: + m = manifests[0] + f = (m.get("decoded") or {}).get("fields") or {} + out["manifest"] = str(m.get("source", "")) + for k_out, k_in in _MANIFEST_KEYS: + if f.get(k_in) is not None: + out[k_out] = f[k_in] + if "channels" not in out and _list(f.get("channelIds")): + out["channels"] = [str(c) for c in _list(f.get("channelIds"))] + if volumes: + image["channels"] = str(len(first_store_of)) + for store in first_store_of.values(): + if not any(_store_of(s) == store for s in sources): + sources.append(store) + elif out.get("channels") and not image["channels"]: + image["channels"] = str(len(out["channels"])) + return out diff --git a/tests/test_edge.py b/tests/test_edge.py index bf74597..bf7b71f 100644 --- a/tests/test_edge.py +++ b/tests/test_edge.py @@ -8,6 +8,7 @@ import base64 import json +from typing import Any from pathlib import Path import scigantic_bil as bil @@ -82,7 +83,8 @@ def test_prior_records_without_image_rows_still_carry_the_instrument() -> None: def test_submission_draft_from_smartspim_records(tmp_path: Path) -> None: index = tmp_path / "index.jsonl" - index.write_text("\n".join(json.dumps(r) for r in [PLANE, PLANE, SMARTSPIM, PLANE]) + '\n{"torn', encoding="utf-8") + planes = [dict(PLANE, source=PLANE["source"].replace("000000", f"{i:06d}")) for i in range(3)] + index.write_text("\n".join(json.dumps(r) for r in [planes[0], planes[1], SMARTSPIM, planes[2]]) + '\n{"torn', encoding="utf-8") draft = submission_draft(tmp_path) assert set(draft["image"]) == set(BIL_IMAGE_COLUMNS) assert set(draft["instrument"]) == set(BIL_INSTRUMENT_COLUMNS) @@ -168,3 +170,234 @@ def test_cli_subcommands(tmp_path: Path, capsys) -> None: # type: ignore[no-unt assert out["run"]["instrument"]["objectivename"] == "LCT 3.6x" assert main(["edge-prior", "ace-cup-eel", str(tmp_path / "p.b64")]) == 0 assert (tmp_path / "p.b64").stat().st_size > 100 + + +# OME-Zarr stores and voxel manifests. voxel_run1_index.jsonl is the +# scigantic-edge decoder's own index.jsonl over a real headless run of the +# Allen Institute voxel engine's simulated instrument (2 XY tiles x 2 +# single-channel profiles, 10 frames of 128 x 192 uint16, 2026-10-20): one +# voxel-manifest record and four ome-zarr records, each carrying the +# manifest's view of that store under fields.manifest. The exaSPIM records +# are hand-written to the same contract from two public tiles' metadata on +# s3://aind-open-data (Zarr v2 / NGFF 0.4 and Zarr v3 / NGFF 0.5, tczyx). +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "edge" +VOXEL_RUN1 = FIXTURES / "voxel_run1_index.jsonl" +EXASPIM_PUBLIC = FIXTURES / "exaspim_public_index.jsonl" + + +def _records(path: Path) -> list[dict]: # type: ignore[type-arg] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def _voxel_parts() -> tuple[dict, list[dict]]: # type: ignore[type-arg] + recs = _records(VOXEL_RUN1) + manifests = [r for r in recs if r["decoded"]["format"] == "voxel-manifest"] + stores = [r for r in recs if r["decoded"]["format"] == "ome-zarr"] + assert len(manifests) == 1 and len(stores) == 4 + return manifests[0], stores + + +def test_voxel_acquisition_is_one_submission() -> None: + drafts = submission_drafts(VOXEL_RUN1) + assert len(drafts) == 1 + (root,) = drafts + assert root.endswith("/run1/store/run1") # the directory holding manifest.json + d = submission_draft(VOXEL_RUN1) + img, inst, acq = d["image"], d["instrument"], d["acquisition"] + assert (img["stepsizex"], img["stepsizey"], img["stepsizez"]) == ("0.47", "0.47", "1") + assert img["channels"] == "2" # gfp + rfp across volumes, not the sizeC=1 of one store + assert (img["slices"], img["zsize"], img["ysize"], img["xsize"]) == ("10", "10", "128", "192") + assert img["dimensionorder"] == "CZYX" + assert acq["status"] == "completed" and acq["revision"] == 15 + assert acq["stores"] == 4 and acq["volumes"] == 4 and acq["volumesPlanned"] == 4 and acq["volumesCompleted"] == 4 + assert acq["channels"] == ["gfp", "rfp"] and acq["manifest"].endswith("/run1/store/run1/manifest.json") + assert acq["startedAt"] < acq["endedAt"] + # The template instrument is a simulator: no light-sheet claim, model as named, + # no objective name/NA or camera model in the record, so those stay blank. + assert inst["microscopetype"] == "" and inst["microscopemanufacturerandmodel"] == "sim" + assert inst["objectivemagnification"] == "8" and inst["objectivena"] == "" and inst["objectivename"] == "" + assert inst["illuminationwavelength"] == "488, 561 nm" and inst["illuminationtypes"] == "laser" + assert inst["detectionwavelength"] == "510, 600 nm" + assert inst["detectormodel"] == "" and inst["detectortype"] == "" + assert d["formats"] == {"voxel-manifest": 1, "ome-zarr": 4} + assert d["sources"][0].endswith("/manifest.json") + assert {s.rsplit("/", 1)[-1] for s in d["sources"][1:]} == {"gfp.ome.zarr", "rfp.ome.zarr"} + + +def test_voxel_stores_group_under_their_manifest_by_each_rule() -> None: + manifest, stores = _voxel_parts() + root = manifest["source"].rsplit("/", 1)[0] + # Rule 1: the sidecar ../../../../manifest.json resolved against the store lands on the + # manifest record, also when the record is keyed on the store's zarr.json. + keyed = json.loads(json.dumps(stores)) + for r in keyed: + r["source"] += "/zarr.json" + assert set(submission_drafts([manifest] + keyed)) == {root} + assert submission_draft([manifest] + keyed)["image"]["channels"] == "2" + # Rule 2: no sidecars; the manifest's storePaths (relative to its directory) name the stores. + no_sidecar = json.loads(json.dumps(stores)) + for r in no_sidecar: + r["decoded"]["fields"]["sidecars"] = [] + assert set(submission_drafts([manifest] + no_sidecar)) == {root} + stale = json.loads(json.dumps(manifest)) + stale["decoded"]["fields"]["storePaths"] = [] + stale["source"] = root + "/elsewhere/manifest.json" + # Neither rule 1 nor 2 pairs the stores with a manifest in another directory: rule 3 keeps + # them together on their own, under the /volumes/ prefix, and the stray manifest is its own draft. + assert set(submission_drafts([stale] + no_sidecar)) == {root, root + "/elsewhere"} + # Rule 3 alone: no manifest record at all yet. Channel ids still come from each store's + # own manifest view, the counts from the stores. + only_stores = submission_drafts(no_sidecar) + assert set(only_stores) == {root} + d = only_stores[root] + assert d["image"]["channels"] == "2" and d["image"]["slices"] == "10" + assert d["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"], "storeStates": {"complete": 4}} + assert d["instrument"]["microscopemanufacturerandmodel"] == "" + # Without the per-store manifest view either, the .ome.zarr names and volume + # directories carry the same counts. + bare = json.loads(json.dumps(no_sidecar)) + for r in bare: + r["decoded"]["fields"].pop("manifest", None) + assert submission_draft(bare)["acquisition"] == {"stores": 4, "volumes": 4, "channels": ["gfp", "rfp"], "storeStates": {"complete": 4}} + # A voxel run under a different root is a different draft, not a blend. + other = json.loads(json.dumps(no_sidecar[:1])) + other[0]["source"] = other[0]["source"].replace("/run1/volumes/", "/run2/volumes/") + assert len(submission_drafts(no_sidecar + other)) == 2 + + +def test_voxel_manifest_alone_seeds_the_draft_before_stores_arrive() -> None: + manifest, _ = _voxel_parts() + d = submission_draft([manifest]) + assert d["image"]["channels"] == "2" and d["image"]["stepsizex"] == "" and d["image"]["slices"] == "" + assert d["acquisition"]["status"] == "completed" and d["acquisition"]["stores"] == 0 + assert "volumes" not in d["acquisition"] and d["acquisition"]["channels"] == ["gfp", "rfp"] + assert d["instrument"]["illuminationwavelength"] == "488, 561 nm" + + +def test_voxel_manifest_light_sheet_only_when_named() -> None: + manifest = json.loads(json.dumps(_voxel_parts()[0])) + manifest["decoded"]["fields"]["instrumentName"] = "exaSPIM-1" + inst = submission_draft([manifest])["instrument"] + assert inst["microscopetype"] == "Light-sheet" and inst["microscopemanufacturerandmodel"] == "exaSPIM-1" + manifest["decoded"]["fields"]["instrumentName"] = "confocal-2" + assert submission_draft([manifest])["instrument"]["microscopetype"] == "" + + +def test_public_exaspim_stores_map_without_a_manifest() -> None: + drafts = submission_drafts(EXASPIM_PUBLIC) + assert set(drafts) == {"exaSPIM_851474_2026-07-10_15-23-22/SPIM.ome.zarr", "exaSPIM_2026-08-04_10-27-41/SPIM"} + v2 = drafts["exaSPIM_851474_2026-07-10_15-23-22/SPIM.ome.zarr"] + img = v2["image"] + assert (img["stepsizex"], img["stepsizey"], img["stepsizez"]) == ("5.984", "5.984", "8") + assert (img["channels"], img["slices"], img["ysize"], img["xsize"]) == ("1", "2560", "1330", "1774") + assert img["dimensionorder"] == "TCZYX" + assert all(v == "" for v in v2["instrument"].values()) + assert v2["acquisition"] == {"stores": 1, "volumes": 1, "channels": ["488"], "storeStates": {"unknown": 1}} + assert v2["formats"] == {"ome-zarr": 1} + v3 = drafts["exaSPIM_2026-08-04_10-27-41/SPIM"] + assert (v3["image"]["stepsizex"], v3["image"]["stepsizez"], v3["image"]["slices"]) == ("15.04", "20", "1024") + assert v3["sources"] == ["exaSPIM_2026-08-04_10-27-41/SPIM/tile_000000_ch_488.ome.zarr/zarr.json"] + + +def test_ome_zarr_voxel_size_units_reach_micrometres() -> None: + from scigantic_bil.edge import _um + + assert _um(0.47, "micrometer") == 0.47 and _um(0.47, "um") == 0.47 + assert _um(0.47, "\u00b5m") == 0.47 and _um(0.47, "\u03bcm") == 0.47 # micro sign, Greek mu + assert _um(0.47, "microns") == 0.47 and _um(0.47, "micrometres") == 0.47 + assert _um(470, "nanometer") == pytest.approx(0.47) and _um(470, "nm") == pytest.approx(0.47) + assert _um(0.001, "millimeter") == pytest.approx(1.0) and _um(0.001, "mm") == pytest.approx(1.0) + assert _um(2, None) == 2.0 and _um(None, "micrometer") is None + rec = json.loads(json.dumps(_records(EXASPIM_PUBLIC)[0])) + f = rec["decoded"]["fields"] + f["voxelSizeUnit"], f["voxelSizeX"], f["voxelSizeY"], f["voxelSizeZ"] = "nanometer", 5984, 5984, 8000 + img = submission_draft([rec])["image"] + assert (img["stepsizex"], img["stepsizez"]) == ("5.984", "8") + + +def test_draft_without_stores_has_an_empty_acquisition_block() -> None: + assert submission_draft([SMARTSPIM])["acquisition"] == {} + + +# --- found by stress-testing the bridge against the edge binary's own output --- + +def _store(source: str, **fields: Any) -> dict[str, Any]: + base = {"layout": "multiscales", "voxelSizeUnit": "micrometer", "state": "complete", "axes": [ + {"name": n} for n in "czyx"]} + base.update(fields) + return {"source": source, "sizeBytes": 1, "mtime": 0, "decoded": {"format": "ome-zarr", "summary": "", "fields": base}} + + +def test_watch_mode_rerecords_keep_only_the_latest_line(tmp_path: Path) -> None: + """index.jsonl is an append log; a store re-recorded as its shards land + must count once and draft from its last state, not its first.""" + early = _store("acq/volumes/000001/p/gfp.ome.zarr", state="partial", sizeZ=4, sizeY=8, sizeX=8, voxelSizeX=1, voxelSizeY=1, voxelSizeZ=2) + late = _store("acq/volumes/000001/p/gfp.ome.zarr", state="complete", sizeZ=40, sizeY=8, sizeX=8, voxelSizeX=1, voxelSizeY=1, voxelSizeZ=2) + index = tmp_path / "index.jsonl" + index.write_text("\n".join(json.dumps(r) for r in [early, early, late]) + "\n", encoding="utf-8") + d = submission_draft(index) + assert d["acquisition"]["stores"] == 1 and d["acquisition"]["storeStates"] == {"complete": 1} + assert d["image"]["slices"] == "40" + assert submission_draft([early, late])["image"]["slices"] == "40" + + +def test_container_group_joins_the_draft_of_its_tiles() -> None: + """AIND's SPIM.ome.zarr is recorded as a group and entered; it must not + split the acquisition into an empty draft of its own.""" + group = _store("ds/SPIM.ome.zarr", layout="group", state="no-array", axes=[]) + tile = _store("ds/SPIM.ome.zarr/tile_000000_ch_488.zarr", sizeZ=10, sizeY=20, sizeX=30, voxelSizeX=1, voxelSizeY=1, voxelSizeZ=1) + drafts = submission_drafts([group, tile]) + assert list(drafts) == ["ds/SPIM.ome.zarr"] + assert drafts["ds/SPIM.ome.zarr"]["acquisition"]["stores"] == 1 + + +def test_channels_at_different_resolutions_are_not_blended() -> None: + """The 2026-08-04 public exaSPIM acquisition stores 561 nm at half the + 488 nm resolution; the Image row must describe one real store and the + disagreement must be reported, not averaged into a row that is neither.""" + fine = _store("a/SPIM/tile_000000_ch_488.ome.zarr", sizeZ=1024, sizeY=2660, sizeX=3548, voxelSizeX=15.04, voxelSizeY=15.04, voxelSizeZ=20) + coarse = _store("a/SPIM/tile_000000_ch_561.ome.zarr", sizeZ=512, sizeY=1330, sizeX=1774, voxelSizeX=30.08, voxelSizeY=30.08, voxelSizeZ=40) + shapeless = _store("a/SPIM/tile_000001_ch_561.ome.zarr", state="no-array", voxelSizeX=30.08, voxelSizeY=30.08, voxelSizeZ=40) + for order in ([coarse, shapeless, fine], [shapeless, fine, coarse]): + d = submission_draft(order) + img = d["image"] + assert (img["stepsizex"], img["stepsizez"], img["xsize"], img["slices"]) == ("15.04", "20", "3548", "1024") + assert d["acquisition"]["mismatch"]["voxelSizeX"] == [15.04, 30.08] + assert d["acquisition"]["mismatch"]["sizeZ"] == [512, 1024] + assert img["channels"] == "2" and d["acquisition"]["channels"] == ["488", "561"] + assert d["acquisition"]["volumes"] == 2 + + +def test_aind_tile_names_count_tiles_and_channels() -> None: + stores = [_store(f"r/SPIM.ome.zarr/tile_x_000{x}_y_0000_z_0000_ch_{ch}.zarr", sizeZ=5, sizeY=5, sizeX=5, + voxelSizeX=0.748, voxelSizeY=0.748, voxelSizeZ=1) for x in range(3) for ch in ("488", "638")] + d = submission_draft(stores) + assert d["acquisition"]["volumes"] == 3 and d["acquisition"]["channels"] == ["488", "638"] + assert d["image"]["channels"] == "2" + + +def test_ome_zarr_scale_without_a_unit_is_not_called_micrometres() -> None: + d = submission_draft([_store("x/a.zarr", voxelSizeUnit=None, voxelSizeX=0.5, voxelSizeY=0.5, voxelSizeZ=2, sizeX=4, sizeY=4, sizeZ=4)]) + assert d["image"]["stepsizex"] == "" and d["image"]["xsize"] == "4" + d = submission_draft([_store("x/a.zarr", voxelSizeUnit="pixel", voxelSizeX=0.5, voxelSizeY=0.5, voxelSizeZ=2, sizeX=4, sizeY=4, sizeZ=4)]) + assert d["image"]["stepsizex"] == "" + d = submission_draft([_store("x/a.zarr", voxelSizeUnit="nanometer", voxelSizeX=500, voxelSizeY=500, voxelSizeZ=2000, sizeX=4, sizeY=4, sizeZ=4)]) + assert (d["image"]["stepsizex"], d["image"]["stepsizez"]) == ("0.5", "2") + + +def test_malformed_records_are_skipped_or_read_as_blank() -> None: + """A torn or hand-edited index must never crash a draft: fields that + are not objects, lists that are scalars, NaN, infinity and huge ints.""" + bad: list[Any] = [ + "not a record", 7, None, {"source": "a/x.zarr"}, {"source": "a/y.zarr", "decoded": []}, + {"source": "a/z.zarr", "decoded": {"format": "ome-zarr", "fields": [1, 2]}}, + {"source": 5, "decoded": {"format": None, "fields": None}}, + {"source": "a/manifest.json", "decoded": {"format": "voxel-manifest", "fields": { + "storePaths": True, "channelIds": 3, "laserWavelengthsNm": 488, "volumeCount": float("inf")}}}, + _store("a/volumes/000001/p/g.ome.zarr", sizeX=float("inf"), sizeY=float("nan"), sizeZ=10**40, + voxelSizeX="NaN", voxelSizeY=None, voxelSizeZ=[], axes=5, sidecars=1, manifest=[]), + ] + drafts = submission_drafts(bad) + json.dumps(drafts) + img = drafts["a"]["image"] + assert img["xsize"] == "" and img["ysize"] == "" and img["stepsizex"] == ""