From 2ec8846b56032246ccbc54622a1eb81e165a9689 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 18:41:15 +0000 Subject: [PATCH 01/16] V_delta: drop redundant status fields; restore v1 session_in_a_dataset shape Driven by the B-corpus discovery report (12,917 v1 docs across 18 classes, 8708 quarantined under the previous schemas). All six classes the corpus surfaces as newly affected are addressed here. Design principle confirmed: V_delta documents are immutable, so "did this thing happen?" tracking fields are redundant -- the existence of the document IS the state. Five required fields that violated this are dropped. Dropped fields: epochfiles_ingested.ingestion_status (required) epochfiles_ingested.num_files_ingested (derived metadata) daqreader_mfdaq_epochdata_ingested.ingestion_status (required) daqmetadatareader_epochdata_ingested.ingestion_status (required) syncrule_mapping.mapping_status (required) dataset_remote.remote_url (required) After the drops these classes either have zero own fields (marker-style records whose presence is the entire signal) or retain only their genuinely-content fields (e.g., syncrule_mapping keeps `mapping_data`; dataset_remote keeps `remote_type` and `dataset_id`). session_in_a_dataset rebuilt to mirror v1 verbatim: before: required `dataset_id` field + required depends_on[session_id] after: session_id (did_uid, required, inline) + session_reference + is_linked + session_creator + session_creator_input1..6 The "dataset_id" concept is intrinsically `base.session_id` in v1's storage model -- the document is found inside a dataset, and the dataset's identity is the dataset's session-id, which lives on every contained document's base block. So the V_delta-only `dataset_id` field was duplicating base.session_id without adding new information. Restoring the full v1 field set also recovers the session-reconstitution metadata (session_creator, session_creator_input1..6, session_reference, is_linked) that the earlier V_delta draft had stripped. Projection on the three discovery corpora (Python simulator): PRED 14/14 migrated (no change) 20211116 1220/1220 migrated (no change) B 12917/12917 migrated (was 4209/12917) No paired did-matlab change is required: the converter's universal-rename pass plus the dispatcher empty-block pad already handle every v1 doc that previously quarantined here. The existing testCorpus* tests continue to gate PRED at zero quarantine and run 20211116 in discovery mode; a third corpus fixture (B) can be added on the did-matlab side as a separate follow-up. --- .../daqmetadatareader_epochdata_ingested.json | 18 +-- .../daqreader_mfdaq_epochdata_ingested.json | 18 +-- schemas/V_delta/stable/dataset_remote.json | 15 -- .../V_delta/stable/epochfiles_ingested.json | 31 +--- .../V_delta/stable/session_in_a_dataset.json | 138 ++++++++++++++++-- schemas/V_delta/stable/syncrule_mapping.json | 15 -- 6 files changed, 131 insertions(+), 104 deletions(-) diff --git a/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json b/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json index 7a387fe..5de9d0c 100644 --- a/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json +++ b/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json @@ -24,21 +24,5 @@ } ], "file": [], - "fields": [ - { - "name": "ingestion_status", - "type": "char", - "blank_value": "", - "default_value": "complete", - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The status of the ingestion ('complete', 'partial', 'failed').", - "constraints": { - "maxLength": 32 - } - } - ] + "fields": [] } diff --git a/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json b/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json index cbba6f2..b0f0d7a 100644 --- a/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json +++ b/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json @@ -24,21 +24,5 @@ } ], "file": [], - "fields": [ - { - "name": "ingestion_status", - "type": "char", - "blank_value": "", - "default_value": "complete", - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The status of the ingestion ('complete', 'partial', 'failed').", - "constraints": { - "maxLength": 32 - } - } - ] + "fields": [] } diff --git a/schemas/V_delta/stable/dataset_remote.json b/schemas/V_delta/stable/dataset_remote.json index c98cb49..830d8a6 100644 --- a/schemas/V_delta/stable/dataset_remote.json +++ b/schemas/V_delta/stable/dataset_remote.json @@ -12,21 +12,6 @@ "depends_on": [], "file": [], "fields": [ - { - "name": "remote_url", - "type": "char", - "blank_value": "", - "default_value": "", - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The URL of the remote dataset location.", - "constraints": { - "maxLength": 1024 - } - }, { "name": "remote_type", "type": "char", diff --git a/schemas/V_delta/stable/epochfiles_ingested.json b/schemas/V_delta/stable/epochfiles_ingested.json index ade29e4..5ec2bef 100644 --- a/schemas/V_delta/stable/epochfiles_ingested.json +++ b/schemas/V_delta/stable/epochfiles_ingested.json @@ -18,34 +18,5 @@ } ], "file": [], - "fields": [ - { - "name": "ingestion_status", - "type": "char", - "blank_value": "", - "default_value": "complete", - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The status of the epoch files ingestion ('complete', 'partial', 'failed').", - "constraints": { - "maxLength": 32 - } - }, - { - "name": "num_files_ingested", - "type": "integer", - "blank_value": 0, - "default_value": 0, - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": true, - "ontology": null, - "documentation": "The number of files ingested for this epoch.", - "constraints": {} - } - ] + "fields": [] } diff --git a/schemas/V_delta/stable/session_in_a_dataset.json b/schemas/V_delta/stable/session_in_a_dataset.json index b59c808..c737a5c 100644 --- a/schemas/V_delta/stable/session_in_a_dataset.json +++ b/schemas/V_delta/stable/session_in_a_dataset.json @@ -9,30 +9,148 @@ ], "maturity_level": "stable" }, - "depends_on": [ + "depends_on": [], + "file": [], + "fields": [ { "name": "session_id", + "type": "did_uid", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": true, - "documentation": "The document ID of the session.", - "must_refer_to_document_class": "" - } - ], - "file": [], - "fields": [ + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": { + "node": "iao:0000578", + "name": "centrally registered identifier" + }, + "documentation": "Document ID of the session referenced by this record. Note: this is the *referenced* session, not the dataset's session (which lives on base.session_id and serves as the dataset identifier).", + "constraints": {} + }, { - "name": "dataset_id", + "name": "session_reference", "type": "char", "blank_value": "", "default_value": "", - "mustBeNonEmpty": true, + "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The identifier of the dataset this session belongs to.", + "documentation": "Human-readable label for the session (e.g., a recording date string like '2008-05-22').", "constraints": { "maxLength": 256 } + }, + { + "name": "is_linked", + "type": "integer", + "blank_value": 0, + "default_value": 0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": true, + "queryable": true, + "ontology": null, + "documentation": "1 if the session is linked into the dataset (i.e., its underlying directory or remote is currently reachable); 0 if it is only referenced by metadata.", + "constraints": { + "min": 0, + "max": 1 + } + }, + { + "name": "session_creator", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Fully-qualified name of the NDI session class that produced this session (e.g., 'ndi.session.dir', 'ndi.session.cloud'). Used together with session_creator_input1..6 to reconstitute the session object.", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "session_creator_input1", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "First positional argument passed to session_creator when reconstituting the session.", + "constraints": {} + }, + { + "name": "session_creator_input2", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Second positional argument passed to session_creator when reconstituting the session.", + "constraints": {} + }, + { + "name": "session_creator_input3", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Third positional argument passed to session_creator when reconstituting the session.", + "constraints": {} + }, + { + "name": "session_creator_input4", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Fourth positional argument passed to session_creator when reconstituting the session.", + "constraints": {} + }, + { + "name": "session_creator_input5", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Fifth positional argument passed to session_creator when reconstituting the session.", + "constraints": {} + }, + { + "name": "session_creator_input6", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Sixth positional argument passed to session_creator when reconstituting the session.", + "constraints": {} } ] } diff --git a/schemas/V_delta/stable/syncrule_mapping.json b/schemas/V_delta/stable/syncrule_mapping.json index 64c5f7a..fa8c6a7 100644 --- a/schemas/V_delta/stable/syncrule_mapping.json +++ b/schemas/V_delta/stable/syncrule_mapping.json @@ -38,21 +38,6 @@ "documentation": "Structure representing the time mapping between clocks for this epoch. Keys are implementation-defined by the sync rule; typically includes the clock names being mapped and the corresponding sample timestamps.", "constraints": {}, "fields": [] - }, - { - "name": "mapping_status", - "type": "char", - "blank_value": "", - "default_value": "complete", - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The status of the mapping computation ('complete', 'failed').", - "constraints": { - "maxLength": 32 - } } ] } From f4db872a7a17d1181f26298a6498eba478586b7d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 19:46:01 +0000 Subject: [PATCH 02/16] V_delta: element_epoch -> array-of-records `clocks` field Driven by the JH corpus surfacing the multi-clock case: v1 documents that record the same element epoch in multiple clock frames (e.g., `epoch_clock = "dev_local_time,exp_global_time"`, `t0_t1 = [[a,b],[c,d]]`). The previous schema required mustBeScalar epoch_clock/t0/t1 and could not represent it. Replaces the three flat scalar fields with a single `element_epoch.clocks` array-of-records: clocks(i).name char required per-clock identifier clocks(i).t0 double required start time in clocks(i) units clocks(i).t1 double required stop time in clocks(i) units Single-clock documents (PRED, 20211116, B corpora) migrate to a 1-element array; multi-clock JH documents migrate to a 2+ element array. mustBeScalar on the parent clocks field is `false`; mustBeNonEmpty is `true` so empty/absent timing data still quarantines. The queryable flag stays on `clocks` so query callers can join via sidecar array-element rows. No paired ngrid/epochclocktimes changes here. The epochclocktimes superclass (sibling of element_epoch with the same scalar-field shape) keeps its current schema for this PR -- so far it's only used as a *superclass* on classes whose v1 form has the single-clock pattern (pyraview in PRED is the only test). If a multi-clock case for an epochclocktimes-using class surfaces, we do the same array-of-records flip there too. Projection on the 20211116, B, and JH corpora after this commit (paired with the did-matlab migrator update): PRED 14/14 migrated (no change) 20211116 1220/1220 migrated (no change) B 12917/12917 migrated (no change) JH 67172/78688 migrated (was 63016; element_epoch +4156) --- schemas/V_delta/stable/element_epoch.json | 87 +++++++++++++---------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/schemas/V_delta/stable/element_epoch.json b/schemas/V_delta/stable/element_epoch.json index 601fbaf..0a09d1b 100644 --- a/schemas/V_delta/stable/element_epoch.json +++ b/schemas/V_delta/stable/element_epoch.json @@ -23,45 +23,60 @@ "file": [], "fields": [ { - "name": "epoch_clock", - "type": "char", - "blank_value": "", - "default_value": "dev_local_time", + "name": "clocks", + "type": "structure", + "blank_value": [], + "default_value": [], "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The clock type that describes this epoch (e.g., 'dev_local_time', 'utc', 'exp_global_time').", - "constraints": { - "maxLength": 256 - } - }, - { - "name": "t0", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, - "mustBeNonEmpty": true, - "mustBeScalar": true, + "mustBeScalar": false, "mustNotHaveNaN": true, - "queryable": false, - "ontology": null, - "documentation": "The start time of this epoch, in time units of epoch_clock.", - "constraints": {} - }, - { - "name": "t1", - "type": "double", - "blank_value": 1.0, - "default_value": 1.0, - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, + "queryable": true, "ontology": null, - "documentation": "The stop time of this epoch, in time units of epoch_clock.", - "constraints": {} + "documentation": "Array-of-records: one entry per clock reference frame in which this element's epoch is timed. Each entry carries the clock identifier (`name`) and the start/stop times (`t0`, `t1`) in that clock's units. The single-clock case (most v1 documents) is a 1-element array; v1 documents that recorded the same epoch in multiple clocks (e.g., 'dev_local_time' + 'exp_global_time') become multi-element arrays. Comma-separated v1 `epoch_clock` strings split on `,` to populate clocks[i].name; v1's N×2 `t0_t1` splits row-wise to populate clocks[i].t0 and clocks[i].t1.", + "constraints": {}, + "fields": [ + { + "name": "name", + "type": "char", + "blank_value": "", + "default_value": "dev_local_time", + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Clock identifier (e.g., 'dev_local_time', 'utc', 'exp_global_time').", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "t0", + "type": "double", + "blank_value": 0.0, + "default_value": 0.0, + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Start time of this epoch in the units of the clock named in this record.", + "constraints": {} + }, + { + "name": "t1", + "type": "double", + "blank_value": 1.0, + "default_value": 1.0, + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Stop time of this epoch in the units of the clock named in this record.", + "constraints": {} + } + ] } ] } From 135ab95c6764cb3d300087211308f5672d88d82d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 12:25:35 +0000 Subject: [PATCH 03/16] V_delta: position_metadata as semantic descriptor, not coordinates The JH corpus surfaces 2078 v1 position_metadata documents, all purely descriptor data (no x/y/z values, no files block). The V_delta draft had inverted the v1 intent by storing concrete numeric coordinates instead of the ontology-driven shape v1 actually carried. Aligned with the design used by `probe_location` elsewhere in V_delta (`location` field as an ontology_term). Rewrites position_metadata to mirror the v1 shape: measurement (ontology_term) required The v1 `ontologyNode` CURIE verbatim, paired with a human- readable name resolved via ndi.ontology.lookup. Classifies *what kind of position* the linked element records (e.g., a midpoint position, a probe tip). units (ontology_term) required The v1 `units` CURIE; same lookup convention as measurement. dimensions (structure-array) optional One record per spatial axis. Records carry an explicit `axis` identifier (defaults to positional `axis_1`, `axis_2`, ...) so queries can filter by axis without resolving the ontology, plus `node` (CURIE) and `name` (resolved label). v1 stored dimensions as a comma-separated CURIE list with implicit positional ordering; this schema preserves that ordering and adds the explicit axis tag. Adds a `depends_on[element_id]` since v1 documents always carry that link (was missing on the previous draft). Migrator-side changes land in the paired did-matlab commit. Projection on JH after this commit (paired with the did-matlab migrator): 76257/78688 migrated (was 74179; position_metadata +2078). Remaining JH quarantine: distance_metadata (2078) and subject_group (353), same class-by-class follow-up. --- schemas/V_delta/stable/position_metadata.json | 134 +++++++++++------- 1 file changed, 82 insertions(+), 52 deletions(-) diff --git a/schemas/V_delta/stable/position_metadata.json b/schemas/V_delta/stable/position_metadata.json index 2e7b85a..774ca50 100644 --- a/schemas/V_delta/stable/position_metadata.json +++ b/schemas/V_delta/stable/position_metadata.json @@ -13,80 +13,110 @@ { "name": "element_id", "mustBeNonEmpty": true, - "documentation": "The document ID of the element whose position is described.", + "documentation": "The document ID of the element whose position is described by this metadata.", "must_refer_to_document_class": "" } ], "file": [], "fields": [ { - "name": "x", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, + "name": "measurement", + "type": "ontology_term", + "blank_value": { + "node": "", + "name": "" + }, + "default_value": { + "node": "", + "name": "" + }, "mustBeNonEmpty": true, "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, - "ontology": null, - "documentation": "The x-coordinate of the element position.", - "constraints": {} - }, - { - "name": "y", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, - "ontology": null, - "documentation": "The y-coordinate of the element position.", - "constraints": {} - }, - { - "name": "z", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, - "ontology": null, - "documentation": "The z-coordinate of the element position.", + "mustNotHaveNaN": false, + "queryable": true, + "ontology": { + "node": "schema:location", + "name": "location" + }, + "documentation": "Ontology term classifying what kind of position is recorded by the linked element (e.g., a midpoint position, an anatomical landmark, a probe tip). The `node` is the v1 `ontologyNode` CURIE verbatim; `name` is resolved via ndi.ontology.lookup at conversion time.", "constraints": {} }, { - "name": "position_units", - "type": "char", - "blank_value": "", - "default_value": "um", + "name": "units", + "type": "ontology_term", + "blank_value": { + "node": "", + "name": "" + }, + "default_value": { + "node": "", + "name": "" + }, "mustBeNonEmpty": true, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The units for the position coordinates (e.g., 'um', 'mm', 'cm').", - "constraints": { - "maxLength": 32 - } + "documentation": "Ontology term naming the measurement unit for the position values (e.g., 'micrometer'). Resolved via ndi.ontology.lookup.", + "constraints": {} }, { - "name": "coordinate_system", - "type": "char", - "blank_value": "", - "default_value": "", + "name": "dimensions", + "type": "structure", + "blank_value": [], + "default_value": [], "mustBeNonEmpty": false, - "mustBeScalar": true, + "mustBeScalar": false, "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The coordinate system or reference frame for this position.", - "constraints": { - "maxLength": 256 - } + "documentation": "Array of per-axis records describing each spatial dimension of the position measurement. v1 stored dimensions as a comma-separated CURIE list with implicit positional ordering; V_delta makes the ordering explicit via the `axis` field while preserving the ontology classification. The migrator labels axes as `axis_1`, `axis_2`, ... by default; an explicit `x`/`y`/`z` convention can be applied per-document where the spatial mapping is known.", + "constraints": {}, + "fields": [ + { + "name": "axis", + "type": "char", + "blank_value": "", + "default_value": "axis_1", + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Per-record axis identifier. Defaults to positional labels (`axis_1`, `axis_2`, ...); consumers may rewrite to a spatial convention (e.g., `x`, `y`, `z`) when known.", + "constraints": { + "maxLength": 64 + } + }, + { + "name": "node", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Ontology CURIE classifying this axis (e.g., a directional or anatomical-axis ontology node).", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "name", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Human-readable label resolved from `node` via ndi.ontology.lookup at conversion time. Empty when the lookup is unavailable.", + "constraints": {} + } + ] } ] } From 1bf2b0d05fb88abdc8e0790a8146ec8928b37ec8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 12:38:22 +0000 Subject: [PATCH 04/16] V_delta: distance_metadata as paired-endpoint records The JH corpus surfaces 2078 v1 distance_metadata documents whose content is paired A/B endpoint metadata, not a scalar distance. Each endpoint carries an ontology classification, a set of integer indices, a comma-separated did_uid list, and an optional numeric values vector; the document records the distance-measurement schema between the two endpoint sets, not the distances themselves. Rewrites distance_metadata to match the v1 shape using the same array-of-records pattern as position_metadata.dimensions: endpoints: structure-array endpoints(i).label (char, required) Per-endpoint identifier preserved verbatim from v1: 'A', 'B'. Mirrors position_metadata's explicit axis labels so queries can filter by endpoint without resolving ontology. endpoints(i).measurement (ontology_term, required) v1 ontologyNode_X CURIE + ndi.ontology.lookup name. endpoints(i).integer_ids (matrix of integer, optional) v1 integerIDs_X verbatim. Often a scalar for endpoint A, a multi-element vector for endpoint B. endpoints(i).string_ids (string array, optional) v1 ontologyStringValues_X parsed (comma-split) into a string array. Each entry is typically a did_uid pointing at another document. endpoints(i).numeric_values (matrix of double, optional) v1 ontologyNumericValues_X. Often empty in the corpora seen so far; preserved for forward compatibility. units (ontology_term, required) Replaces the previous schema's `distance_units` char. v1 already stored this as a CURIE. The previous schema's `distance` scalar and depends_on edges (`element_id_1`/`element_id_2`) are dropped: v1 had neither. depends_on becomes a single `element_id` matching the v1 idiom. Projected JH after this commit (paired with the did-matlab migrator): 78335/78688 migrated. Remaining: subject_group (353) and Dab stimulus_bath (1605), unchanged. --- schemas/V_delta/stable/distance_metadata.json | 125 ++++++++++++++---- 1 file changed, 100 insertions(+), 25 deletions(-) diff --git a/schemas/V_delta/stable/distance_metadata.json b/schemas/V_delta/stable/distance_metadata.json index 2a6701b..01f581e 100644 --- a/schemas/V_delta/stable/distance_metadata.json +++ b/schemas/V_delta/stable/distance_metadata.json @@ -11,47 +11,122 @@ }, "depends_on": [ { - "name": "element_id_1", + "name": "element_id", "mustBeNonEmpty": true, - "documentation": "The document ID of the first element.", - "must_refer_to_document_class": "" - }, - { - "name": "element_id_2", - "mustBeNonEmpty": true, - "documentation": "The document ID of the second element.", + "documentation": "The document ID of the element whose endpoints' distance is described by this metadata.", "must_refer_to_document_class": "" } ], "file": [], "fields": [ { - "name": "distance", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, + "name": "endpoints", + "type": "structure", + "blank_value": [], + "default_value": [], "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": true, "ontology": null, - "documentation": "The distance between the two elements, in distance_units.", - "constraints": {} + "documentation": "Array of per-endpoint records describing the entities whose pairwise distance is being measured. v1 stored exactly two endpoints labelled 'A' and 'B'; V_delta keeps those labels explicit on each record's `label` field but generalises the container to an array so future N-endpoint distance schemes can reuse the same shape. Each endpoint carries: an ontology classification (`measurement`), a set of integer indices and document-id strings identifying which items at the linked element the endpoint covers, and an optional numeric-values vector. v1's `ontologyStringValues_X` (comma-separated did_uid list) becomes `string_ids` as a string array; v1's `integerIDs_X` (scalar or array) becomes `integer_ids` as a matrix.", + "constraints": {}, + "fields": [ + { + "name": "label", + "type": "char", + "blank_value": "", + "default_value": "A", + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Per-endpoint identifier preserved verbatim from v1 (e.g., 'A', 'B'). Consumers can filter by label without resolving the ontology classification.", + "constraints": { + "maxLength": 64 + } + }, + { + "name": "measurement", + "type": "ontology_term", + "blank_value": { + "node": "", + "name": "" + }, + "default_value": { + "node": "", + "name": "" + }, + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Ontology term classifying what kind of entity this endpoint refers to (e.g., a probe contact, a stimulation site). `node` is the v1 `ontologyNode_X` CURIE verbatim; `name` is resolved via ndi.ontology.lookup at conversion time.", + "constraints": {} + }, + { + "name": "integer_ids", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Per-endpoint integer indices (e.g., contact numbers, item indices). v1 stored this as a scalar or array of integers; V_delta keeps it as a flat matrix of integers.", + "constraints": { + "element_type": "integer" + } + }, + { + "name": "string_ids", + "type": "string", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Per-endpoint document-id strings identifying the items the endpoint covers. v1 stored this as a single comma-separated did_uid string; V_delta represents it as a string array (one entry per id). Indexed in parallel with `integer_ids` when both are populated.", + "constraints": {} + }, + { + "name": "numeric_values", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Optional per-endpoint numeric values (often empty in v1 corpora; reserved for calibrations or per-item measurements that the endpoint definition needs to carry inline).", + "constraints": {} + } + ] }, { - "name": "distance_units", - "type": "char", - "blank_value": "", - "default_value": "um", + "name": "units", + "type": "ontology_term", + "blank_value": { + "node": "", + "name": "" + }, + "default_value": { + "node": "", + "name": "" + }, "mustBeNonEmpty": true, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The units for the distance value (e.g., 'um', 'mm', 'cm').", - "constraints": { - "maxLength": 32 - } + "documentation": "Ontology term naming the measurement unit for the distance (e.g., 'micrometer'). Resolved via ndi.ontology.lookup.", + "constraints": {} } ] } From a164e7cba9372de5f86f8573cb276ee61ac6f189 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 12:48:27 +0000 Subject: [PATCH 05/16] V_delta: subject_group as a depends_on-only marker class The JH corpus surfaces 353 v1 subject_group documents whose property block is universally empty (`{}`) and whose base.name is also universally empty -- the class is a pure relational marker. Subject membership is recorded via depends_on edges (`subject_id_1`, `subject_id_2`, ...). The previous V_delta draft required a `group_name` v1 never recorded and added a `subject_ids` char field that duplicated depends_on as a delimited string. Changes: - `group_name` mustBeNonEmpty: true -> false. v1 docs migrate with this field absent; new documents may populate it for ad-hoc labeling. - `description` unchanged (already optional). - `subject_ids` field removed entirely. The depends_on array already carries typed `subject_id_N` edges; a parallel comma-separated char field is redundant and worse than the structured form. No migrator change needed. v1 subject_group bodies flow through universalRenames unchanged (empty block stays empty; depends_on entries are renamed from id->value). Projection on JH after this commit: 78688/78688 migrated. Remaining quarantine: only Dab stimulus_bath (1605). --- schemas/V_delta/stable/subject_group.json | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/schemas/V_delta/stable/subject_group.json b/schemas/V_delta/stable/subject_group.json index 22a9217..fb2de07 100644 --- a/schemas/V_delta/stable/subject_group.json +++ b/schemas/V_delta/stable/subject_group.json @@ -17,7 +17,7 @@ "type": "char", "blank_value": "", "default_value": "", - "mustBeNonEmpty": true, + "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, @@ -25,7 +25,7 @@ "node": "schema:name", "name": "name" }, - "documentation": "The name of the subject group (e.g., 'control', 'treatment'). Candidate for promotion to ontology_term once a study-arm vocabulary (e.g., NCIT 'Study Arm' branch) is registered.", + "documentation": "Optional human-readable name for this subject group (e.g., 'control', 'treatment'). v1 docs did not carry a name and migrate with this field absent; new documents may populate it for ad-hoc labeling. Candidate for promotion to ontology_term once a study-arm vocabulary (e.g., NCIT 'Study Arm' branch) is registered.", "constraints": { "maxLength": 256 } @@ -43,26 +43,10 @@ "node": "schema:description", "name": "description" }, - "documentation": "A free-text description of this subject group.", + "documentation": "Optional free-text description of this subject group.", "constraints": { "maxLength": 1024 } - }, - { - "name": "subject_ids", - "type": "char", - "blank_value": "", - "default_value": "", - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": false, - "ontology": { - "node": "iao:0000578", - "name": "centrally registered identifier" - }, - "documentation": "Comma-separated list of subject document IDs belonging to this group (e.g., 'id1,id2,id3').", - "constraints": {} } ] } From 7f53d90a83da4e3df4474c8949482dd0cfc64197 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 13:03:27 +0000 Subject: [PATCH 06/16] V_delta: stimulus_bath redesign + new `concentration` composite type Driven by the Dab corpus (1605 v1 stimulus_bath documents). v1 stores an ontology-typed bath location plus an inline CSV table of chemicals, each with its own ontology classification and concentration. The previous V_delta draft assumed one solution name plus a scalar concentration -- it can't represent v1's multi-chemical baths. ## New composite type: concentration The other SI composites (duration / volume / mass / length / voltage / current / frequency) all share one canonical sub-field (meters, volts, ...) because their source units convert to that canonical by a single scalar. Concentration does not have that property: mass-per-volume cannot be converted to molar without molecular weight, and vice versa. Forcing a single canonical would make any concentration that ships without MW uninterpretable. `concentration` therefore has multiple OPTIONAL canonical sub-fields, with the migrator populating whichever the source unit is computable into: molar (double, opt) mol/L grams_per_liter (double, opt) mass/volume mass_fraction (double, opt) w/w (dimensionless 0-1) volume_fraction (double, opt) v/v (dimensionless 0-1) approximate (boolean) source_unit (char) verbatim source unit text source_value (double) verbatim source value Added to the meta-schema type enum and described in the top-level meta-schema documentation alongside the existing SI composites. The did-matlab validator switch is updated in a paired commit to accept the new type (same isstruct check as other composites). ## stimulus_bath redesign Replaces solution_name/concentration/concentration_units with: super: [base, epochid] (epochid restored to match v1) depends_on: [stimulus_element_id] (renamed from element_id) fields: * location (ontology_term, REQ) the bath itself mixture (structure, array-of-records): chemical (ontology_term, REQ) amount (concentration, opt) The migrator parses the v1 CSV mixture_table (header row + chemicals), each row producing a record with chemical (node+name from v1 ontologyName/name) and amount (concentration composite from v1 value/unitName). Projection after this commit (paired with did-matlab migrator): PRED 14/14 migrated unchanged 20211116 1220/1220 migrated unchanged B 12917/12917 migrated unchanged JH 78688/78688 migrated unchanged Dab 27561/27561 migrated (was 25956; +1605 stimulus_bath) All five discovery corpora round-trip clean. See did-schema review issue #46 for the detailed conversion table and the full design rationale. --- schemas/V_delta/stable/did_schema_meta.json | 5 +- schemas/V_delta/stable/stimulus_bath.json | 108 ++++++++++++-------- 2 files changed, 71 insertions(+), 42 deletions(-) diff --git a/schemas/V_delta/stable/did_schema_meta.json b/schemas/V_delta/stable/did_schema_meta.json index 1c5d3ed..76f68ce 100644 --- a/schemas/V_delta/stable/did_schema_meta.json +++ b/schemas/V_delta/stable/did_schema_meta.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://did-schema.example.org/V_delta/did_schema_meta.json", "title": "DID/NDI Schema Meta-Schema (V_delta)", - "description": "Validates the structure of DID/NDI schema files for V_delta. V_delta introduces named composite types 'duration', 'ontology_term', 'volume', 'mass', 'length', 'voltage', 'current', and 'frequency', and a redesigned field-level 'ontology' annotation shape with keys 'node' (CURIE) and 'name' (label); the CURIE prefix is resolved against CURIE_lookups_meta.json. The six SI-dimensioned types share the same sub-field layout as 'duration' (a canonical-unit double, 'approximate' boolean, 'source_unit' char, 'source_value' double); only the canonical sub-field name and the allowed source units differ. NAMING CONVENTION: V_delta drops the V_beta underscore prefix on NDI-extension keys. Every reserved key \u2014 JSON Schema vocabulary or NDI extension \u2014 is used without an underscore prefix; the authoritative enumeration of NDI-reserved names lives in ndi_reserved_keys.json and is enforced by this meta-schema. Schema authors must not reuse a reserved name as their own data field name.", + "description": "Validates the structure of DID/NDI schema files for V_delta. V_delta introduces named composite types 'duration', 'ontology_term', 'volume', 'mass', 'length', 'voltage', 'current', 'frequency', and 'concentration', and a redesigned field-level 'ontology' annotation shape with keys 'node' (CURIE) and 'name' (label); the CURIE prefix is resolved against CURIE_lookups_meta.json. The six SI-dimensioned types ('duration', 'volume', 'mass', 'length', 'voltage', 'current', 'frequency') share the same sub-field layout (a canonical-unit double, 'approximate' boolean, 'source_unit' char, 'source_value' double); only the canonical sub-field name and the allowed source units differ. 'concentration' breaks the single-canonical pattern because concentration units do not collapse to a single canonical (mass/volume cannot be converted to molar without molecular weight, and vice versa); instead it offers multiple OPTIONAL canonical sub-fields ('molar', 'grams_per_liter', 'mass_fraction', 'volume_fraction') and the migrator populates whichever the source unit is computable into, alongside the same 'approximate' / 'source_unit' / 'source_value' triple. NAMING CONVENTION: V_delta drops the V_beta underscore prefix on NDI-extension keys. Every reserved key \u2014 JSON Schema vocabulary or NDI extension \u2014 is used without an underscore prefix; the authoritative enumeration of NDI-reserved names lives in ndi_reserved_keys.json and is enforced by this meta-schema. Schema authors must not reuse a reserved name as their own data field name.", "type": "object", "required": [ "document_class", @@ -227,7 +227,8 @@ "length", "voltage", "current", - "frequency" + "frequency", + "concentration" ], "description": "Data type of the field. Uses the standard JSON Schema keyword 'type'; values are NDI-specific." }, diff --git a/schemas/V_delta/stable/stimulus_bath.json b/schemas/V_delta/stable/stimulus_bath.json index e6fb884..5101fa6 100644 --- a/schemas/V_delta/stable/stimulus_bath.json +++ b/schemas/V_delta/stable/stimulus_bath.json @@ -5,71 +5,99 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "epochid" } ], "maturity_level": "stable" }, "depends_on": [ { - "name": "element_id", + "name": "stimulus_element_id", "mustBeNonEmpty": true, - "documentation": "The document ID of the element receiving this bath stimulus.", + "documentation": "The document ID of the stimulus element this bath records.", "must_refer_to_document_class": "" } ], "file": [], "fields": [ { - "name": "solution_name", - "type": "char", - "blank_value": "", - "default_value": "", + "name": "location", + "type": "ontology_term", + "blank_value": { + "node": "", + "name": "" + }, + "default_value": { + "node": "", + "name": "" + }, "mustBeNonEmpty": true, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, "ontology": { - "node": "iao:0000219", - "name": "denotes" - }, - "documentation": "The name of the bath solution (e.g., 'ACSF', 'TTX'). The field denotes a chemical entity / drug; candidate for promotion to ontology_term once a chemistry namespace (e.g., ChEBI, DrON) is registered.", - "constraints": { - "maxLength": 256 - } - }, - { - "name": "concentration", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": false, - "ontology": { - "node": "pato:0000033", - "name": "concentration of" + "node": "schema:location", + "name": "location" }, - "documentation": "The concentration of the active compound.", + "documentation": "Ontology term naming the type of bath the stimulus was applied in (e.g., NCIm:C0179246, 'Baths, Water, Laboratory'). v1 already stored this as an ontology-term-shaped sub-object; the migrator renames `ontologyNode` -> `node` in lockstep with the V_delta ontology_term convention.", "constraints": {} }, { - "name": "concentration_units", - "type": "char", - "blank_value": "", - "default_value": "", + "name": "mixture", + "type": "structure", + "blank_value": [], + "default_value": [], "mustBeNonEmpty": false, - "mustBeScalar": true, + "mustBeScalar": false, "mustNotHaveNaN": false, "queryable": true, - "ontology": { - "node": "iao:0000003", - "name": "measurement unit label" - }, - "documentation": "The units of the concentration (e.g., 'mM', 'uM', 'nM').", - "constraints": { - "maxLength": 32 - } + "ontology": null, + "documentation": "Array of per-chemical records describing the chemical mixture the bath delivered. v1 stored this as an inline CSV (`mixture_table`) with header row `ontologyName,name,value,ontologyUnit,unitName`; V_delta parses each data row into a structured record. Each record carries the chemical identity as an ontology_term and the concentration as a `concentration` composite (multiple optional canonical sub-fields plus source-unit/source-value tracking).", + "constraints": {}, + "fields": [ + { + "name": "chemical", + "type": "ontology_term", + "blank_value": { + "node": "", + "name": "" + }, + "default_value": { + "node": "", + "name": "" + }, + "mustBeNonEmpty": true, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Ontology term identifying this chemical species. `node` comes from v1 `ontologyName`; `name` comes from v1 `name`.", + "constraints": {} + }, + { + "name": "amount", + "type": "concentration", + "blank_value": { + "approximate": false, + "source_unit": "", + "source_value": 0.0 + }, + "default_value": { + "approximate": false, + "source_unit": "", + "source_value": 0.0 + }, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Concentration of this chemical in the bath. Carries source_unit / source_value (v1 `unitName` and `value`) and zero or more canonical sub-fields (`molar`, `grams_per_liter`, `mass_fraction`, `volume_fraction`) populated whenever the source unit is computable into them. For v1 docs in the Dab corpus the source unit is uniformly 'Molar' so `molar = source_value`.", + "constraints": {} + } + ] } ] } From ad577060564c5a3bc30b4709f7eff001a84c0aaa Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 13:17:34 +0000 Subject: [PATCH 07/16] V_delta: metadata_editor matches v1 dataset-descriptor shape The Soph corpus surfaces a single v1 metadata_editor document whose content does not match the previous V_delta draft. v1 uses the class to store dataset-level descriptors (VersionIdentifier, License, DataType, etc.); the previous draft modeled it as "metadata about an editor tool" (editor_class + target_classname), which v1 never recorded. Rewrites metadata_editor to match v1 exactly: super: [base] fields: metadata_structure (structure, optional) The single open-shape `metadata_structure` field carries the arbitrary key/value pairs v1 stored; keys and inner shape are intentionally not constrained by V_delta since they vary by editor and dataset. The dropped `editor_class` and `target_classname` fields had no v1 source. Projection on Soph after this commit: 101427/101427 migrated. All six discovery corpora round-trip clean. --- schemas/V_delta/stable/metadata_editor.json | 32 ++++++--------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/schemas/V_delta/stable/metadata_editor.json b/schemas/V_delta/stable/metadata_editor.json index 4a4d63e..2c44cc9 100644 --- a/schemas/V_delta/stable/metadata_editor.json +++ b/schemas/V_delta/stable/metadata_editor.json @@ -13,34 +13,18 @@ "file": [], "fields": [ { - "name": "editor_class", - "type": "char", - "blank_value": "", - "default_value": "", - "mustBeNonEmpty": true, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": true, - "ontology": null, - "documentation": "The class name of the metadata editor implementation.", - "constraints": { - "maxLength": 256 - } - }, - { - "name": "target_classname", - "type": "char", - "blank_value": "", - "default_value": "", + "name": "metadata_structure", + "type": "structure", + "blank_value": {}, + "default_value": {}, "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, - "queryable": true, + "queryable": false, "ontology": null, - "documentation": "The DID classname of the documents this editor can modify.", - "constraints": { - "maxLength": 256 - } + "documentation": "Arbitrary key/value metadata describing the dataset this document was authored against. v1 documents in the Soph corpus carry dataset-level descriptors here (VersionIdentifier, License, DataType, and others). The shape is intentionally open: keys and inner structure depend on the editor and the dataset.", + "constraints": {}, + "fields": [] } ] } From e57702573c4f9d0b5391971dbb7b3a944bbabd50 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 13:50:39 +0000 Subject: [PATCH 08/16] V_delta: declare tuningcurve_calc.log and stim_property_list Driven by the strict-validation audit (did-matlab commit aaf5529): every v1 tuningcurve_calc body in the discovery corpora ships two fields V_delta did not declare, so the data was landing in the property block as undeclared extras and the strict validator quarantined them with did2:validation:undeclaredField. The two fields are tuningcurve_calc-specific (no other *_calc class in any of the six discovery corpora ships them), so they land directly on tuningcurve_calc rather than being promoted to the calculator base. tuningcurve_calc.log (char, optional) Free-text log entry summarising the calculation, e.g. 'angle best value is 135.', 'sFrequency = 0.04'. tuningcurve_calc.stim_property_list (structure, optional) Stimulus-property name/value pairs that conditioned this tuning curve. v1 shape preserved verbatim: names (string array, optional) - e.g. {'sFrequency'} values (matrix, optional) - corresponding scalar/array Cleared corpus quarantines: Soph 34606 tuningcurve_calc docs 20211116 84 tuningcurve_calc docs total 34690 fewer undeclared-field errors The next biggest cluster surfacing in the strict-mode report is the v1 `stimulus_tuningcurve` inheritance on tuningcurve_calc, which V_delta currently drops. Separate follow-up. --- schemas/V_delta/stable/tuningcurve_calc.json | 60 ++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/schemas/V_delta/stable/tuningcurve_calc.json b/schemas/V_delta/stable/tuningcurve_calc.json index 1bbec92..e401cd4 100644 --- a/schemas/V_delta/stable/tuningcurve_calc.json +++ b/schemas/V_delta/stable/tuningcurve_calc.json @@ -37,6 +37,66 @@ "dynamic_keys_from": "calculator_name" }, "fields": [] + }, + { + "name": "log", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Free-text log entry summarising the tuning-curve calculation (e.g., 'angle best value is 135.', 'sFrequency = 0.04'). Captured verbatim from v1 documents that carried a `log` field.", + "constraints": {} + }, + { + "name": "stim_property_list", + "type": "structure", + "blank_value": { + "names": [], + "values": [] + }, + "default_value": { + "names": [], + "values": [] + }, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Stimulus-property name/value pairs that conditioned this tuning-curve calculation. v1 stored this as `{names: [, ...], values: }`; V_delta preserves the same shape. `names` is the list of independent stimulus parameter names selected for the curve (most commonly a single name such as 'sFrequency'); `values` is the corresponding value(s).", + "constraints": {}, + "fields": [ + { + "name": "names", + "type": "string", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Array of stimulus parameter names (e.g., {'sFrequency', 'tFrequency'}). Empty array when no stimulus property was held; v1 most often ships a 1-element array.", + "constraints": {} + }, + { + "name": "values", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Numeric value(s) paralleling `names`. v1 ships either a scalar (when names is 1-element) or an empty array.", + "constraints": {} + } + ] } ] } From d60972a8b38451375065061dc3ab791fbeb76da0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 14:14:29 +0000 Subject: [PATCH 09/16] V_delta: declare epoch-ingestion + syncrule-mapping content fields Driven by strict-validation audit (did-matlab aaf5529). When PR #44 dropped the redundant `*_status` fields from these classes, the schemas were left essentially empty -- every v1 content field then surfaced as undeclaredField under strict mode. Restored field declarations to match v1 verbatim: syncrule_mapping: cost (double, opt) numeric mapping score mapping (matrix, opt) 2-element numeric vector epochnode_a (structure) {epoch_clock, epoch_id, epoch_session_id, epochprobemap, objectclass} epochnode_b (structure) same shape as epochnode_a (dropped: mapping_data -- was aspirational, never in v1) epochfiles_ingested: epoch_id (char, opt) epoch identifier (e.g., 't00001') files (string, opt, list of file references; entries are !mustBeScalar) NDI URIs or absolute paths epochprobemap (char, opt) tab-separated probemap text daqreader_mfdaq_epochdata_ingested: parameters (structure, opt) {sample_analog_segment, sample_digital_segment} daqmetadatareader_epochdata_ingested: unchanged (v1 block is empty; strict validator already passes). Projection delta on the discovery corpora (Python simulator): B 1738 -> 4222 migrated (+2484 syncrule_mapping cleared) Dab 10375 -> 12859 migrated (+2484 syncrule_mapping cleared) Soph 36826 -> 37174 migrated (+348 syncrule_mapping cleared) PRED, 20211116, JH: unchanged (these corpora either don't have these classes or have different residual issues). Residual sub-issues for these classes (separate follow-ups): epochfiles_ingested.files declared as type=string mustBeScalar=false, but v1 decodes to a cell array of chars in MATLAB; the validator's string case currently accepts only ischar/isstring, not iscell. Either a one-line validator relaxation or a per-class migrator that converts cell -> string array clears this. daqreader_mfdaq_epochdata_ingested inherits from a v1 parent `daqreader_epochdata_ingested` that V_delta does not declare; the v1 doc carries an inherited block that surfaces as undeclaredBlock. Add the parent class to V_delta (most v1-faithful) or have a migrator drop the inherited block. The simulator over-reports vs real MATLAB on cell-of-chars (real MATLAB also rejects it for type=string today; same fix needed in both places). The corpus CI run will confirm the authoritative numbers. --- .../daqreader_mfdaq_epochdata_ingested.json | 50 +++- .../V_delta/stable/epochfiles_ingested.json | 47 +++- schemas/V_delta/stable/syncrule_mapping.json | 220 +++++++++++++++++- 3 files changed, 309 insertions(+), 8 deletions(-) diff --git a/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json b/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json index b0f0d7a..00a33ad 100644 --- a/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json +++ b/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json @@ -24,5 +24,53 @@ } ], "file": [], - "fields": [] + "fields": [ + { + "name": "parameters", + "type": "structure", + "blank_value": { + "sample_analog_segment": 0.0, + "sample_digital_segment": 0.0 + }, + "default_value": { + "sample_analog_segment": 0.0, + "sample_digital_segment": 0.0 + }, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Reader-specific parameters captured at the time the epoch was ingested. The two sub-fields seen in the v1 corpora are sample-count cutoffs the MFDAQ reader used to slice the source recording.", + "constraints": {}, + "fields": [ + { + "name": "sample_analog_segment", + "type": "double", + "blank_value": 0.0, + "default_value": 0.0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Maximum sample count per analog segment used during ingestion.", + "constraints": {} + }, + { + "name": "sample_digital_segment", + "type": "double", + "blank_value": 0.0, + "default_value": 0.0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Maximum sample count per digital segment used during ingestion.", + "constraints": {} + } + ] + } + ] } diff --git a/schemas/V_delta/stable/epochfiles_ingested.json b/schemas/V_delta/stable/epochfiles_ingested.json index 5ec2bef..2c6bf57 100644 --- a/schemas/V_delta/stable/epochfiles_ingested.json +++ b/schemas/V_delta/stable/epochfiles_ingested.json @@ -18,5 +18,50 @@ } ], "file": [], - "fields": [] + "fields": [ + { + "name": "epoch_id", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": { + "node": "iao:0000578", + "name": "centrally registered identifier" + }, + "documentation": "Epoch identifier within the owning session (e.g., 't00001'). v1-faithful inline copy of the epoch's local name; the document-id link to the epoch lives in `depends_on[epochid]`.", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "files", + "type": "string", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "List of file references ingested for this epoch. Entries are either NDI-scheme URIs (e.g., 'epochid://t00001') or absolute filesystem paths to the source files. Length varies per epoch (3-6 entries seen in the discovery corpora).", + "constraints": {} + }, + { + "name": "epochprobemap", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Tab-separated epoch-probe-map text describing the probe configuration at ingest time. Free-form pass-through from v1; structure is `name\\treference\\ttype\\tdevicestring\\tsubjectstring\\n\\n...`.", + "constraints": {} + } + ] } diff --git a/schemas/V_delta/stable/syncrule_mapping.json b/schemas/V_delta/stable/syncrule_mapping.json index fa8c6a7..49ce6fc 100644 --- a/schemas/V_delta/stable/syncrule_mapping.json +++ b/schemas/V_delta/stable/syncrule_mapping.json @@ -26,18 +26,226 @@ "file": [], "fields": [ { - "name": "mapping_data", - "type": "structure", - "blank_value": {}, - "default_value": {}, + "name": "cost", + "type": "double", + "blank_value": 0, + "default_value": 0, "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Numeric cost / score of this mapping (lower is typically better). v1 ships an integer; V_delta uses `double` to admit fractional costs from future rules.", + "constraints": {} + }, + { + "name": "mapping", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "Structure representing the time mapping between clocks for this epoch. Keys are implementation-defined by the sync rule; typically includes the clock names being mapped and the corresponding sample timestamps.", + "documentation": "Numeric mapping vector relating the two endpoints in clock space. v1 ships a 2-element numeric vector (int or float depending on the rule); V_delta keeps the shape open as a matrix.", + "constraints": {} + }, + { + "name": "epochnode_a", + "type": "structure", + "blank_value": { + "epoch_clock": "", + "epoch_id": "", + "epoch_session_id": "", + "epochprobemap": "", + "objectclass": "" + }, + "default_value": { + "epoch_clock": "", + "epoch_id": "", + "epoch_session_id": "", + "epochprobemap": "", + "objectclass": "" + }, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "First endpoint of the mapping. Identifies the source epoch in clock space. Preserved verbatim from v1; the same shape applies to epochnode_b.", + "constraints": {}, + "fields": [ + { + "name": "epoch_clock", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Clock identifier for this endpoint (e.g., 'dev_local_time').", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "epoch_id", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Epoch identifier within the endpoint's session (e.g., 't00001').", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "epoch_session_id", + "type": "did_uid", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Document id of the session this endpoint's epoch lives in.", + "constraints": {} + }, + { + "name": "epochprobemap", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Tab-separated epoch-probe-map text describing the probe configuration at this endpoint. Free-form pass-through from v1.", + "constraints": {} + }, + { + "name": "objectclass", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Fully-qualified NDI epochprobemap class name (e.g., 'ndi.epoch.epochprobemap_daqsystem').", + "constraints": { + "maxLength": 256 + } + } + ] + }, + { + "name": "epochnode_b", + "type": "structure", + "blank_value": { + "epoch_clock": "", + "epoch_id": "", + "epoch_session_id": "", + "epochprobemap": "", + "objectclass": "" + }, + "default_value": { + "epoch_clock": "", + "epoch_id": "", + "epoch_session_id": "", + "epochprobemap": "", + "objectclass": "" + }, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Second endpoint of the mapping. Same shape as epochnode_a.", "constraints": {}, - "fields": [] + "fields": [ + { + "name": "epoch_clock", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Clock identifier for this endpoint.", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "epoch_id", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Epoch identifier within the endpoint's session.", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "epoch_session_id", + "type": "did_uid", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Document id of the session this endpoint's epoch lives in.", + "constraints": {} + }, + { + "name": "epochprobemap", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Tab-separated epoch-probe-map text. Free-form pass-through from v1.", + "constraints": {} + }, + { + "name": "objectclass", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Fully-qualified NDI epochprobemap class name.", + "constraints": { + "maxLength": 256 + } + } + ] } ] } From 78b9fa75a9eb3dbc9069a5a2db77dc219dac8f16 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 14:36:35 +0000 Subject: [PATCH 10/16] V_delta: stimulus_presentation declares stimuli + restores app/epochid Driven by strict-validation audit. Three changes: 1. Declare `stimuli` (structure, optional, mustBeScalar=true) on stimulus_presentation. v1 carries it on every doc as `{parameters: {...stimulus-type-specific keys...}}` -- the parameters sub-shape depends on the stimulus generator (Hartley basis, sparse-noise grid, oriented gratings) and on the generating library version. V_delta declares `stimuli.parameters` but does not constrain inner keys. 2. Drop `num_trials` (integer field). v1 never ships it; was aspirational. 3. Restore v1 superclasses dropped by the previous V_delta draft: base + app + epochid v1 stimulus_presentation documents carry app provenance metadata (NDI calculator name + version + interpreter) and an epochid linking to the trial epoch. The previous draft only declared base, so the multi-inheritance walk in the strict validator reported `app` and `epochid` blocks as undeclared. `presentation_order` is also slightly loosened: dropped the `element_type: integer` constraint and the integer-array-only documentation. v1 corpora ship a scalar 1 in every doc seen, not a per-trial vector; treating it as an open matrix is more faithful. Projection delta (Python simulator) on the four corpora that ship stimulus_presentation docs: 20211116 542 -> 553 migrated (+11) B 4222 -> 5464 migrated (+1242) Dab 12859 -> 14101 migrated (+1242) Soph 37174 -> 37349 migrated (+175) Total: 2670 stimulus_presentation docs now migrate cleanly under strict validation. No paired did-matlab change required. --- .../V_delta/stable/stimulus_presentation.json | 53 +++++++++++++------ 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/schemas/V_delta/stable/stimulus_presentation.json b/schemas/V_delta/stable/stimulus_presentation.json index 53e0525..1dde021 100644 --- a/schemas/V_delta/stable/stimulus_presentation.json +++ b/schemas/V_delta/stable/stimulus_presentation.json @@ -5,6 +5,12 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "app" + }, + { + "class_name": "epochid" } ], "maturity_level": "stable" @@ -32,26 +38,41 @@ "node": "placeholder:stimulus_presentation_order", "name": "stimulus presentation order" }, - "documentation": "Array of integer stimulus indices (1-based) in the order they were presented. Length equals num_trials.", - "constraints": { - "element_type": "integer" - } + "documentation": "Stimulus presentation index/order. v1 commonly ships a scalar 1; consumers that need a per-trial array fill it as a 1-based integer vector whose length equals the number of trials.", + "constraints": {} }, { - "name": "num_trials", - "type": "integer", - "blank_value": 0, - "default_value": 0, + "name": "stimuli", + "type": "structure", + "blank_value": { + "parameters": {} + }, + "default_value": { + "parameters": {} + }, "mustBeNonEmpty": false, "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": true, - "ontology": { - "node": "placeholder:trial_count", - "name": "trial count" - }, - "documentation": "The total number of stimulus trials.", - "constraints": {} + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Stimulus configuration captured at presentation time. The sole sub-field that all v1 corpora ship is `parameters`, an open-shape struct whose keys depend on the stimulus type (Hartley basis, sparse-noise grid, oriented gratings, etc.) and on the generating library version. V_delta does not constrain inner field names because each stimulus type contributes its own; the validator only verifies that `stimuli` is itself a struct. Consumers that need to interpret specific parameter values should branch on the stimulus class context (e.g., the linked `app.app_name`) before reading.", + "constraints": {}, + "fields": [ + { + "name": "parameters", + "type": "structure", + "blank_value": {}, + "default_value": {}, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Stimulus-type-specific parameter object. Open-shape: keys and value types depend on the stimulus generator. v1 corpora include Hartley-basis (rect, windowShape, M, K_absmax, randState, ...), sparse-noise (BG, value, random, repeat, pixSize, randState, ...), and many other variants.", + "constraints": {}, + "fields": [] + } + ] } ] } From 1fbba099273e526015f0cbab5d2964406660f7fd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 20:09:32 +0000 Subject: [PATCH 11/16] V_delta: neuron_extracellular declares all v1 fields + app superclass Same pattern as stimulus_presentation: v1 ships 7 content fields plus an `app` superclass, V_delta drafted 4 different fields and declared only base as a superclass. Strict-validation audit surfaced 1511 docs across 20211116 and Soph hitting undeclaredField / undeclaredBlock failures. v1 ships uniformly across all 1511 docs: cluster_index int number_of_channels int number_of_samples_per_channel int mean_waveform matrix (samples x channels) waveform_sample_times matrix quality_label char (e.g., 'good', 'multi') quality_number int (sorter-specific grade) [superclasses: base, app] V_delta now declares all of these and lists `app` as a superclass so v1's app block (provenance: ndi.spike_sorter app name + version + interpreter) validates against the app schema. Dropped from the previous V_delta draft (no v1 source): quality (replaced by quality_label, which is the v1 spelling) num_spikes (derivable from waveform/spiketimes data downstream) mean_firing_rate (derivable; was annotated as a `frequency` composite candidate but no v1 doc ships it) Projection delta (Python simulator): 20211116 553 -> 574 migrated (+21 neuron_extracellular) Soph 37349 -> 38839 migrated (+1490 neuron_extracellular) Total 1511 neuron_extracellular docs now migrate cleanly under strict validation. No paired did-matlab change required. --- .../V_delta/stable/neuron_extracellular.json | 86 +++++++++++++------ 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/schemas/V_delta/stable/neuron_extracellular.json b/schemas/V_delta/stable/neuron_extracellular.json index 1a44258..2186be2 100644 --- a/schemas/V_delta/stable/neuron_extracellular.json +++ b/schemas/V_delta/stable/neuron_extracellular.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "app" } ], "maturity_level": "stable" @@ -32,11 +35,63 @@ "node": "iao:0000578", "name": "centrally registered identifier" }, - "documentation": "The cluster index assigned to this neuron by a spike sorter. Matches the did_v1 field name; an earlier V_delta draft renamed it to `cluster_id` but the v1 source uses `cluster_index`, so the original name is preserved.", + "documentation": "The cluster index assigned to this neuron by a spike sorter.", + "constraints": {} + }, + { + "name": "number_of_channels", + "type": "integer", + "blank_value": 0, + "default_value": 0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": true, + "queryable": true, + "ontology": null, + "documentation": "Number of recording channels contributing to this unit's waveform (e.g., the tetrode/probe channel count).", "constraints": {} }, { - "name": "quality", + "name": "number_of_samples_per_channel", + "type": "integer", + "blank_value": 0, + "default_value": 0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": true, + "queryable": true, + "ontology": null, + "documentation": "Number of time samples per channel in mean_waveform / waveform_sample_times.", + "constraints": {} + }, + { + "name": "mean_waveform", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Mean spike-waveform matrix shaped (number_of_samples_per_channel x number_of_channels) -- samples down rows, channels across columns. Recorded in the original A/D units of the source data.", + "constraints": {} + }, + { + "name": "waveform_sample_times", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Time stamp of each sample in mean_waveform (seconds, in the element's local clock); length equals number_of_samples_per_channel.", + "constraints": {} + }, + { + "name": "quality_label", "type": "char", "blank_value": "", "default_value": "", @@ -48,13 +103,13 @@ "node": "placeholder:spike_sorting_quality_label", "name": "spike-sorting quality label" }, - "documentation": "Quality label for this unit (e.g., 'good', 'mua', 'noise'). Candidate for promotion to ontology_term once a controlled vocabulary (e.g., a spike-sorting quality branch in NIFSTD) is registered.", + "documentation": "Human-readable spike-sorting quality label (e.g., 'good', 'multi', 'noise'). Candidate for promotion to ontology_term once a controlled vocabulary (e.g., a spike-sorting quality branch in NIFSTD) is registered.", "constraints": { "maxLength": 64 } }, { - "name": "num_spikes", + "name": "quality_number", "type": "integer", "blank_value": 0, "default_value": 0, @@ -62,27 +117,8 @@ "mustBeScalar": true, "mustNotHaveNaN": true, "queryable": true, - "ontology": { - "node": "placeholder:action_potential_count", - "name": "action potential count" - }, - "documentation": "The total number of spikes assigned to this neuron.", - "constraints": {} - }, - { - "name": "mean_firing_rate", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": false, - "queryable": false, - "ontology": { - "node": "pato:0000044", - "name": "frequency" - }, - "documentation": "The mean firing rate of this neuron in Hz. Candidate for promotion to the V_gamma 'frequency' composite type.", + "ontology": null, + "documentation": "Numeric spike-sorting quality grade (e.g., 1-4) paired with quality_label. Sorter-specific scale.", "constraints": {} } ] From 6a47212a124dbae69c759a60c49850f15b1a441d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 20:17:43 +0000 Subject: [PATCH 12/16] V_delta: openminds family declares v1 fields + restores inheritance Same pattern as stimulus_presentation / neuron_extracellular. Strict-validation audit surfaced 11448 openminds-family docs hitting undeclaredField (openminds itself) and undeclaredBlock (its subclasses) failures. v1 design: openminds (8 docs, JH) fields: openminds_type (URL IRI), matlab_type (MATLAB class name), openminds_id (instance IRI), fields (open-shape struct, openminds-type-specific) openminds_subject (10401 docs, Dab + JH) block: {} -- empty; superclass=[base, openminds] openminds_element (404 docs, Dab) block: {} -- empty; superclass=[base, openminds] openminds_stimulus (635 docs, Dab) block: {} -- empty; superclass=[base, openminds, epochid] Changes: openminds -- field set rewritten to match v1 verbatim: + matlab_type, openminds_id, fields (struct, open) = openminds_type (was already there; doc string updated to reflect the full IRI v1 ships rather than the short `core.Person` style note) - openminds_data, openminds_version (no v1 source) openminds_subject -- add `openminds` as superclass. openminds_element -- add `openminds` as superclass. openminds_stimulus -- add `openminds` and `epochid` as superclasses. Subclass blocks stay empty (v1-faithful: the rich content lives in the inherited openminds block; subclasses are marker types identifying what kind of entity the openminds metadata describes). Projection delta (Python simulator): Dab 14101 -> 16445 migrated (+2344 openminds_*) JH 62584 -> 71624 migrated (+9040 openminds + openminds_subject) Soph 38839 -> 38903 migrated (+64 openminds_subject) Total: +11448 openminds-family docs cleared (matches the audit count exactly). --- schemas/V_delta/stable/openminds.json | 48 ++++++++++++------- schemas/V_delta/stable/openminds_element.json | 3 ++ .../V_delta/stable/openminds_stimulus.json | 6 +++ schemas/V_delta/stable/openminds_subject.json | 3 ++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/schemas/V_delta/stable/openminds.json b/schemas/V_delta/stable/openminds.json index 14106c1..0c1891f 100644 --- a/schemas/V_delta/stable/openminds.json +++ b/schemas/V_delta/stable/openminds.json @@ -22,41 +22,57 @@ "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The openMINDS schema type (e.g., 'core.DatasetVersion', 'core.Person').", + "documentation": "Full openMINDS type IRI identifying which openMINDS schema this document carries (e.g., 'https://openminds.om-i.org/types/GeneticStrainType').", "constraints": { - "maxLength": 256 + "maxLength": 1024 } }, { - "name": "openminds_data", - "type": "structure", - "blank_value": {}, - "default_value": {}, + "name": "matlab_type", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, - "queryable": false, + "queryable": true, "ontology": null, - "documentation": "Structure containing the openMINDS metadata fields conforming to the schema type specified in openminds_type. Keys and value types follow the openMINDS specification for that type.", + "documentation": "Fully-qualified MATLAB class name of the corresponding openMINDS object (e.g., 'openminds.controlledterms.GeneticStrainType'). Paired with openminds_type for tooling that needs to reconstitute the MATLAB object.", "constraints": { - "conforms_to": "openminds_type" - }, - "fields": [] + "maxLength": 512 + } }, { - "name": "openminds_version", + "name": "openminds_id", "type": "char", "blank_value": "", - "default_value": "v3", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, - "ontology": null, - "documentation": "The version of the openMINDS specification used.", + "ontology": { + "node": "iao:0000578", + "name": "centrally registered identifier" + }, + "documentation": "openMINDS instance IRI (e.g., 'https://openminds.om-i.org/instances/geneticStrainType/wildtype'). Identifies a particular openMINDS object instance rather than its type.", "constraints": { - "maxLength": 16 + "maxLength": 1024 } + }, + { + "name": "fields", + "type": "structure", + "blank_value": {}, + "default_value": {}, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Open-shape struct carrying the openMINDS object's payload fields. Keys are openMINDS-type-specific (e.g., GeneticStrainType uses {definition, description, interlexIdentifier, knowledgeSpaceLink, name}); V_delta does not constrain them so each openMINDS type can contribute its own.", + "constraints": {}, + "fields": [] } ] } diff --git a/schemas/V_delta/stable/openminds_element.json b/schemas/V_delta/stable/openminds_element.json index bf50a78..dd903b4 100644 --- a/schemas/V_delta/stable/openminds_element.json +++ b/schemas/V_delta/stable/openminds_element.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "openminds" } ], "maturity_level": "stable" diff --git a/schemas/V_delta/stable/openminds_stimulus.json b/schemas/V_delta/stable/openminds_stimulus.json index f6cdba8..c791cfb 100644 --- a/schemas/V_delta/stable/openminds_stimulus.json +++ b/schemas/V_delta/stable/openminds_stimulus.json @@ -5,6 +5,12 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "openminds" + }, + { + "class_name": "epochid" } ], "maturity_level": "stable" diff --git a/schemas/V_delta/stable/openminds_subject.json b/schemas/V_delta/stable/openminds_subject.json index 201b49a..5efdaf5 100644 --- a/schemas/V_delta/stable/openminds_subject.json +++ b/schemas/V_delta/stable/openminds_subject.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "openminds" } ], "maturity_level": "stable" From 2b4d7afcf6e70f62d1fb37e7693665daccd72cb2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 20:32:33 +0000 Subject: [PATCH 13/16] V_delta: stimulus_tuningcurve declares all 16 v1 fields + tuningcurve_calc inherits it The biggest remaining strict-validation cluster: tuningcurve_calc documents in 20211116 (84) and Soph (34606) carry a v1 `stimulus_tuningcurve` superclass block that V_delta did not declare. The previous V_delta stimulus_tuningcurve schema only declared 4 fields (independent_variable, independent_values, response_mean, response_stderr), but v1 ships 16, with different naming conventions. v1 stimulus_tuningcurve uniformly ships across 34690 docs: independent_variable_label string array independent_variable_value matrix (N_stim x N_var) stimid matrix integer response_mean matrix response_stddev matrix response_stderr matrix response_units string array individual_responses_real matrix (N_trial x N_stim) individual_responses_imaginary matrix (N_trial x N_stim) stimulus_presentation_number matrix integer control_stimid matrix integer control_response_mean matrix control_response_stddev matrix control_response_stderr matrix control_individual_responses_real matrix (N_trial x N_stim) control_individual_responses_imaginary matrix (N_trial x N_stim) Changes: stimulus_tuningcurve -- rewritten to declare all 16 v1 fields verbatim. The previous schema's `independent_variable` and `independent_values` (camel-style aspirational draft) are dropped in favour of the v1-faithful `independent_variable_label` and `independent_variable_value`. response_mean and response_stderr are preserved as-is (already matched). response_stddev, the control_* family, individual_responses_*, stimid, stimulus_ presentation_number, and response_units are added. tuningcurve_calc.superclasses += stimulus_tuningcurve. The v1 inheritance puts stimulus_tuningcurve in the tuningcurve_calc chain; the previous V_delta draft had only base + calculator, so the strict validator flagged the v1 stimulus_tuningcurve block as undeclared. Paired with did-matlab change to relax `type=string` validator to also accept cell-of-chars (MATLAB's jsondecode produces cells for JSON arrays of strings; both `independent_variable_label` here and `epochfiles_ingested.files` need that). Projection delta on the discovery corpora (Python simulator): 20211116 574 -> 658 migrated (+84 tuningcurve_calc) B 5464 -> 7948 migrated (+2484 epochfiles_ingested) Dab 16445 -> 20533 migrated (+4088 epochfiles_ingested + co-resident classes) Soph 38903 -> 73858 migrated (+34955 tuningcurve_calc + epochfiles_ingested) JH 71624 unchanged (no tuningcurve_calc / epochfiles_ingested affected here; JH's remaining quarantines are image_stack and a few small clusters) Single largest schema cleanup so far: ~42K docs cleared across four corpora. --- .../V_delta/stable/stimulus_tuningcurve.json | 193 ++++++++++++++++-- schemas/V_delta/stable/tuningcurve_calc.json | 3 + 2 files changed, 175 insertions(+), 21 deletions(-) diff --git a/schemas/V_delta/stable/stimulus_tuningcurve.json b/schemas/V_delta/stable/stimulus_tuningcurve.json index ac57158..0910d69 100644 --- a/schemas/V_delta/stable/stimulus_tuningcurve.json +++ b/schemas/V_delta/stable/stimulus_tuningcurve.json @@ -20,25 +20,23 @@ "file": [], "fields": [ { - "name": "independent_variable", - "type": "char", + "name": "independent_variable_label", + "type": "string", "blank_value": "", "default_value": "", - "mustBeNonEmpty": true, - "mustBeScalar": true, + "mustBeNonEmpty": false, + "mustBeScalar": false, "mustNotHaveNaN": false, "queryable": true, "ontology": { "node": "obi:0000750", "name": "independent variable" }, - "documentation": "The name of the independent stimulus variable (e.g., 'angle', 'spatial_frequency'). Candidate for promotion to ontology_term.", - "constraints": { - "maxLength": 256 - } + "documentation": "String label(s) for the independent variable(s) the tuning curve was computed across. v1 ships this as a 1-element list whose single entry is a comma-separated list of variable names when more than one variable was crossed (e.g., 'spatial_frequency,temporal_frequency'). V_delta keeps it as a string array; callers split the comma-separated form themselves if needed.", + "constraints": {} }, { - "name": "independent_values", + "name": "independent_variable_value", "type": "matrix", "blank_value": [], "default_value": [], @@ -46,13 +44,23 @@ "mustBeScalar": false, "mustNotHaveNaN": false, "queryable": false, - "ontology": { - "node": "iao:0000027", - "name": "data item" - }, - "documentation": "Array of numeric values of the independent variable, one per tuning curve point. Must be the same length as response_mean and response_stderr.", + "ontology": null, + "documentation": "Matrix of independent-variable values, one row per stimulus condition. Shape is (N_stimuli x N_independent_variables). For a single-variable tuning curve each row is length 1; for the 2D case (e.g., spatial-by-temporal frequency) each row is length 2.", + "constraints": {} + }, + { + "name": "stimid", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Integer stimulus identifiers (1-based) for each row of independent_variable_value. Length equals number of stimulus conditions.", "constraints": { - "element_type": "double" + "element_type": "integer" } }, { @@ -68,10 +76,21 @@ "node": "ncit:C53319", "name": "Arithmetic Mean" }, - "documentation": "Array of mean response values at each point in independent_values. Must be the same length as independent_values.", - "constraints": { - "element_type": "double" - } + "documentation": "Per-stimulus mean of the scalar responses (length equals number of stimuli).", + "constraints": {} + }, + { + "name": "response_stddev", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Per-stimulus standard deviation of the scalar responses.", + "constraints": {} }, { "name": "response_stderr", @@ -86,10 +105,142 @@ "node": "stato:0000037", "name": "standard error of the mean" }, - "documentation": "Array of standard error of the mean for each response value. Must be the same length as independent_values. Each element is itself a SEM; the array dimension is across independent_values.", + "documentation": "Per-stimulus standard error of the mean (length equals number of stimuli).", + "constraints": {} + }, + { + "name": "response_units", + "type": "string", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Units of the scalar response values. v1 often ships an empty array; populated when the source recording's units are known.", + "constraints": {} + }, + { + "name": "individual_responses_real", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Real component of each individual trial response. Shape (N_trials x N_stimuli). Paired with individual_responses_imaginary for complex-valued responses; for real-valued responses individual_responses_imaginary is zero or absent.", + "constraints": {} + }, + { + "name": "individual_responses_imaginary", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Imaginary component of each individual trial response. Same shape as individual_responses_real.", + "constraints": {} + }, + { + "name": "stimulus_presentation_number", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Per-trial, per-stimulus index of the original stimulus presentation. Shape (N_trials x N_stimuli). Lets callers map an individual_responses_real[t,s] back to the original stimulus_presentation document.", "constraints": { - "element_type": "double" + "element_type": "integer" } + }, + { + "name": "control_stimid", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Stimulus identifiers for the control (blank/baseline) condition(s). Often empty when no control was recorded.", + "constraints": { + "element_type": "integer" + } + }, + { + "name": "control_response_mean", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Mean response on the control condition(s). Same length as response_mean (typically broadcasts a single control value across stimuli).", + "constraints": {} + }, + { + "name": "control_response_stddev", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Standard deviation of the response on the control condition(s).", + "constraints": {} + }, + { + "name": "control_response_stderr", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Standard error of the mean for the response on the control condition(s).", + "constraints": {} + }, + { + "name": "control_individual_responses_real", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Real component of individual trial responses on the control condition(s). Same shape conventions as individual_responses_real.", + "constraints": {} + }, + { + "name": "control_individual_responses_imaginary", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Imaginary component of individual trial responses on the control condition(s).", + "constraints": {} } ] } diff --git a/schemas/V_delta/stable/tuningcurve_calc.json b/schemas/V_delta/stable/tuningcurve_calc.json index e401cd4..ede266b 100644 --- a/schemas/V_delta/stable/tuningcurve_calc.json +++ b/schemas/V_delta/stable/tuningcurve_calc.json @@ -8,6 +8,9 @@ }, { "class_name": "calculator" + }, + { + "class_name": "stimulus_tuningcurve" } ], "maturity_level": "stable" From 395b446b1ef41574d5e76604063548f7343e3d32 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 20:38:22 +0000 Subject: [PATCH 14/16] V_delta: image_stack / image_stack_parameters declare v1 fields Strict-validation audit surfaced 7007 image_stack docs in JH all failing on undeclaredField image_stack.label. v1 ships a completely different field set than the V_delta draft, and v1 inherits from imageStack_parameters (camelCase block name that wasn't snake-cased on the v2 side). v1 imageStack ships uniformly: label (char) free-text caption formatOntology (char) ontology CURIE classifying the format [superclasses: base, imageStack_parameters] v1 imageStack_parameters ships uniformly: dimension_order (char) axis order, one char per axis dimension_labels (char) comma-separated per-axis labels dimension_size (matrix) per-axis pixel/sample counts dimension_scale (matrix) per-axis physical scale dimension_scale_units (char) comma-separated per-axis units data_type (char) pixel data type ("uint16", etc.) data_limits (matrix) [min, max] pixel range timestamp (double) acquisition timestamp clocktype (char) clock identifier V_delta drafts had unrelated fields: image_stack: num_frames / x_pixels / y_pixels / image_format image_stack_parameters: z_step / z_units / x_pixel_size / y_pixel_size / pixel_units Rewrites both to match v1 verbatim and adds image_stack_parameters as a superclass of image_stack so the v1 inheritance is honoured. The v1 block-name `imageStack_parameters` (camelCase) is snake-cased to `image_stack_parameters` by the paired did-matlab change to universalRenames (which now snake-cases all top-level block keys, not just the concrete-class key). Projection delta on JH (the only corpus with image_stack): 71624 -> 78631 migrated (+7007 image_stack docs cleared). PRED stays 14/14; other corpora unchanged (no image_stack docs). --- schemas/V_delta/stable/image_stack.json | 47 ++------ .../stable/image_stack_parameters.json | 108 ++++++++++++++---- 2 files changed, 96 insertions(+), 59 deletions(-) diff --git a/schemas/V_delta/stable/image_stack.json b/schemas/V_delta/stable/image_stack.json index 012535e..e992d56 100644 --- a/schemas/V_delta/stable/image_stack.json +++ b/schemas/V_delta/stable/image_stack.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "image_stack_parameters" } ], "maturity_level": "stable" @@ -25,46 +28,20 @@ ], "fields": [ { - "name": "num_frames", - "type": "integer", - "blank_value": 0, - "default_value": 0, + "name": "label", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, - "mustNotHaveNaN": true, + "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The number of frames (slices) in the image stack.", - "constraints": {} - }, - { - "name": "x_pixels", - "type": "integer", - "blank_value": 0, - "default_value": 0, - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, - "ontology": null, - "documentation": "The number of pixels in the x-dimension per frame.", - "constraints": {} - }, - { - "name": "y_pixels", - "type": "integer", - "blank_value": 0, - "default_value": 0, - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": false, - "ontology": null, - "documentation": "The number of pixels in the y-dimension per frame.", + "documentation": "Free-text label or caption describing the image stack (e.g., 'An image of fluorescently labeled bacterial patches that has been processed to ...').", "constraints": {} }, { - "name": "image_format", + "name": "format_ontology", "type": "char", "blank_value": "", "default_value": "", @@ -73,9 +50,9 @@ "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The file format of the image stack (e.g., 'tiff', 'nifti').", + "documentation": "Ontology CURIE describing the image format / content type (e.g., 'NCIT:C70631' for a particular imaging modality). v1 corpora ship the CURIE directly as a char; consumers resolve via ndi.ontology.lookup when a human-readable name is needed.", "constraints": { - "maxLength": 32 + "maxLength": 256 } } ] diff --git a/schemas/V_delta/stable/image_stack_parameters.json b/schemas/V_delta/stable/image_stack_parameters.json index f221c59..ffbbaea 100644 --- a/schemas/V_delta/stable/image_stack_parameters.json +++ b/schemas/V_delta/stable/image_stack_parameters.json @@ -12,7 +12,7 @@ "depends_on": [ { "name": "imagestack_id", - "mustBeNonEmpty": true, + "mustBeNonEmpty": false, "documentation": "The document ID of the image stack these parameters describe.", "must_refer_to_document_class": "" } @@ -20,72 +20,132 @@ "file": [], "fields": [ { - "name": "z_step", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, + "name": "dimension_order", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "String encoding the spatial dimension order of the stack, one character per axis (e.g., 'YX' for a 2-D height-by-width image, 'ZYX' or 'TZYX' for higher-dimensional stacks).", + "constraints": { + "maxLength": 16 + } + }, + { + "name": "dimension_labels", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Comma-separated human-readable labels for each axis (e.g., 'height,width', 'depth,height,width'). One label per character in dimension_order.", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "dimension_size", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": true, + "queryable": false, + "ontology": null, + "documentation": "Pixel/sample count along each axis. Length equals number of characters in dimension_order (e.g., [2208, 2752] for a 'YX' stack).", + "constraints": { + "element_type": "integer" + } + }, + { + "name": "dimension_scale", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "The step size in the z-dimension between frames.", + "documentation": "Physical scale per axis (e.g., 94.89 micrometers per Y pixel). Same length as dimension_size; units captured in dimension_scale_units.", "constraints": {} }, { - "name": "z_units", + "name": "dimension_scale_units", "type": "char", "blank_value": "", - "default_value": "um", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, - "queryable": false, + "queryable": true, "ontology": null, - "documentation": "The units for the z-step (e.g., 'um', 'mm').", + "documentation": "Comma-separated unit names for each axis (e.g., 'micrometer,micrometer', 'micrometer,micrometer,micrometer').", "constraints": { - "maxLength": 32 + "maxLength": 256 } }, { - "name": "x_pixel_size", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, + "name": "data_type", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "The numeric data type of pixel values (e.g., 'uint16', 'single', 'double').", + "constraints": { + "maxLength": 32 + } + }, + { + "name": "data_limits", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "The physical size of a pixel in the x-dimension.", + "documentation": "2-element [min, max] range of valid pixel values for the declared data_type (e.g., [0, 65535] for uint16).", "constraints": {} }, { - "name": "y_pixel_size", + "name": "timestamp", "type": "double", "blank_value": 0.0, "default_value": 0.0, "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, - "queryable": false, + "queryable": true, "ontology": null, - "documentation": "The physical size of a pixel in the y-dimension.", + "documentation": "Acquisition timestamp of this image stack in the units of clocktype.", "constraints": {} }, { - "name": "pixel_units", + "name": "clocktype", "type": "char", "blank_value": "", - "default_value": "um", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, - "queryable": false, + "queryable": true, "ontology": null, - "documentation": "The units for the pixel size (e.g., 'um', 'mm').", + "documentation": "Clock identifier in whose units `timestamp` is expressed (e.g., 'exp_global_time', 'dev_local_time'). Same vocabulary as the epochclocktimes/element_epoch clock names.", "constraints": { - "maxLength": 32 + "maxLength": 256 } } ] From db58d19976f8a3cca27ea0e5bfd14764953d3bc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 May 2026 13:22:54 +0000 Subject: [PATCH 15/16] V_delta: stimulus_response_scalar_parameters_basic + tuning rename Two strict-validation clusters resolved together: 1. stimulus_response_scalar_parameters_basic field rewrite v1 ships uniformly across 11440 docs: temporalfreqfunc (char) freq_response (integer 0/1) prestimulus_time (matrix) prestimulus_normalization (matrix) isspike (integer 0/1) spiketrain_dt (double) [superclasses: base, stimulus_response_scalar_parameters] V_delta drafted: response_window_start, response_window_end, freq_response Rewritten to declare all 6 v1 fields verbatim; dropped the aspirational response_window_* fields. Added stimulus_response_scalar_parameters as a superclass so the v1 inheritance is honoured. 2. spatial_frequency_tuning / temporal_frequency_tuning rename V_delta drafts both had a `fit_sgauss` field; v1 corpora uniformly use `fit_gausslog`. Same field, drafted under a shorter spelling that v1 never adopted. Renamed both occurrences (field name + cross-reference in the `abs` documentation). Projection delta on the corpora that carry these classes: 20211116 658 -> 931 migrated (+273 stimulus_response_*_basic) Soph 73858 -> 90629 migrated (+16771 spatial_/temporal_calc +stimulus_response_*_basic) ~17K docs cleared. JH stays at 78631/57; B and Dab unchanged (no spatial/temporal calc docs in those). --- .../stable/spatial_frequency_tuning.json | 4 +- ...ulus_response_scalar_parameters_basic.json | 85 ++++++++++++++----- .../stable/temporal_frequency_tuning.json | 4 +- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/schemas/V_delta/stable/spatial_frequency_tuning.json b/schemas/V_delta/stable/spatial_frequency_tuning.json index a3771c1..7366a3d 100644 --- a/schemas/V_delta/stable/spatial_frequency_tuning.json +++ b/schemas/V_delta/stable/spatial_frequency_tuning.json @@ -797,7 +797,7 @@ ] }, { - "name": "fit_sgauss", + "name": "fit_gausslog", "type": "structure", "blank_value": {}, "default_value": {}, @@ -931,7 +931,7 @@ "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "Mirror of all preceding blocks (tuning_curve, significance, fitless, fit_dog, fit_movshon, fit_movshon_c, fit_spline, fit_sgauss) recomputed on the absolute value of the responses. Empty in NDIcalc-vis-matlab v1; sub-fields to be populated as the calc implementation matures.", + "documentation": "Mirror of all preceding blocks (tuning_curve, significance, fitless, fit_dog, fit_movshon, fit_movshon_c, fit_spline, fit_gausslog) recomputed on the absolute value of the responses. Empty in NDIcalc-vis-matlab v1; sub-fields to be populated as the calc implementation matures.", "constraints": {}, "fields": [] } diff --git a/schemas/V_delta/stable/stimulus_response_scalar_parameters_basic.json b/schemas/V_delta/stable/stimulus_response_scalar_parameters_basic.json index 0b413fe..aa189ca 100644 --- a/schemas/V_delta/stable/stimulus_response_scalar_parameters_basic.json +++ b/schemas/V_delta/stable/stimulus_response_scalar_parameters_basic.json @@ -5,48 +5,75 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "stimulus_response_scalar_parameters" } ], "maturity_level": "stable" }, - "depends_on": [ - { - "name": "stimulus_response_scalar_id", - "mustBeNonEmpty": true, - "documentation": "The document ID of the stimulus_response_scalar these basic parameters describe.", - "must_refer_to_document_class": "" - } - ], + "depends_on": [], "file": [], "fields": [ { - "name": "response_window_start", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, + "name": "temporalfreqfunc", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Fully-qualified NDI function name that produced the temporal-frequency component of the scalar response (e.g., 'ndi.fun.stimulustemporalfrequency'). Identifies the analysis routine.", + "constraints": { + "maxLength": 256 + } + }, + { + "name": "freq_response", + "type": "integer", + "blank_value": 0, + "default_value": 0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": true, + "queryable": true, + "ontology": null, + "documentation": "1 if this scalar response is a frequency-domain measurement, 0 otherwise. v1 corpora ship integer flags here.", + "constraints": { + "min": 0, + "max": 1 + } + }, + { + "name": "prestimulus_time", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "Start of the response window in seconds after stimulus onset.", + "documentation": "Optional pre-stimulus baseline window. v1 ships either an empty array (no baseline subtraction) or a [start, stop] pair in seconds before stimulus onset.", "constraints": {} }, { - "name": "response_window_end", - "type": "double", - "blank_value": 0.0, - "default_value": 0.0, + "name": "prestimulus_normalization", + "type": "matrix", + "blank_value": [], + "default_value": [], "mustBeNonEmpty": false, - "mustBeScalar": true, + "mustBeScalar": false, "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "End of the response window in seconds after stimulus onset.", + "documentation": "Optional normalization vector applied during the prestimulus baseline computation. v1 commonly ships an empty array; populated when a non-trivial baseline scheme is in use.", "constraints": {} }, { - "name": "freq_response", + "name": "isspike", "type": "integer", "blank_value": 0, "default_value": 0, @@ -55,7 +82,23 @@ "mustNotHaveNaN": true, "queryable": true, "ontology": null, - "documentation": "The frequency component used for the response computation (0 = mean, 1 = F1, 2 = F2).", + "documentation": "1 if the source signal is a spike train (analysed at spiketrain_dt resolution), 0 if it is a continuous signal.", + "constraints": { + "min": 0, + "max": 1 + } + }, + { + "name": "spiketrain_dt", + "type": "double", + "blank_value": 0.0, + "default_value": 0.001, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Time step (seconds) at which a spike train is discretised before computing the scalar response (e.g., 0.001 for 1 ms bins). Only meaningful when isspike == 1.", "constraints": {} } ] diff --git a/schemas/V_delta/stable/temporal_frequency_tuning.json b/schemas/V_delta/stable/temporal_frequency_tuning.json index b620db5..abc8e06 100644 --- a/schemas/V_delta/stable/temporal_frequency_tuning.json +++ b/schemas/V_delta/stable/temporal_frequency_tuning.json @@ -797,7 +797,7 @@ ] }, { - "name": "fit_sgauss", + "name": "fit_gausslog", "type": "structure", "blank_value": {}, "default_value": {}, @@ -931,7 +931,7 @@ "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "Mirror of all preceding blocks (tuning_curve, significance, fitless, fit_dog, fit_movshon, fit_movshon_c, fit_spline, fit_sgauss) recomputed on the absolute value of the responses. Empty in NDIcalc-vis-matlab v1; sub-fields to be populated as the calc implementation matures.", + "documentation": "Mirror of all preceding blocks (tuning_curve, significance, fitless, fit_dog, fit_movshon, fit_movshon_c, fit_spline, fit_gausslog) recomputed on the absolute value of the responses. Empty in NDIcalc-vis-matlab v1; sub-fields to be populated as the calc implementation matures.", "constraints": {}, "fields": [] } From 1d3c8436049cae4aa4227374aa7cbc30b31efd63 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 May 2026 14:20:19 +0000 Subject: [PATCH 16/16] V_delta: close out remaining v1 quarantine reasons across 6 corpora Brings all 6 v1 corpora (PRED, 20211116, B, Dab, JH, Soph; 221,827 docs total) to 100% clean migration with zero quarantines. Per-class changes (all to match v1-faithful shapes the corpora actually ship): - stimulus_response: replace `response_type` with the two v1 fields `stimulator_epochid` and `element_epochid` (response_type already lives on the child stimulus_response_scalar block). - stimulus_response_scalar: pre-existing class header; no field change. - control_stimulus_ids: add `app` superclass (v1 ships a populated app block on these documents). - probe_location: switch from composite `location: ontology_term` back to v1's flat `ontology_name` + `name` chars. - treatment: switch from composite `treatment_name: ontology_term` back to v1's flat `ontology_name` + `name` chars (keeping numeric_value / string_value unchanged). - daqreader_epochdata_ingested: drop ingestion_status marker; add the `epochtable` struct (epochclock string-array + t0_t1 [t0,t1] pair). - daqreader_mfdaq_epochdata_ingested: add `daqreader_epochdata_ingested` and `epochid` to superclasses; drop redundant local depends_on. - daqmetadatareader_epochdata_ingested: add `epochid` superclass. - jrclust_clusters: replace aspirational num_clusters/jrclust_version with v1's `res_mat_md5_checksum`; add `app` superclass. - dataset_remote: add `organization_id` field. - app: relax mustBeNonEmpty on app_name so legacy v1 docs that ship an empty app block (e.g. some jrclust_clusters) still validate. Validation: pytest 96/96 green. --- schemas/V_delta/stable/app.json | 2 +- .../V_delta/stable/control_stimulus_ids.json | 86 +++++++++++++++---- .../daqmetadatareader_epochdata_ingested.json | 9 +- .../stable/daqreader_epochdata_ingested.json | 52 ++++++++--- .../daqreader_mfdaq_epochdata_ingested.json | 21 ++--- schemas/V_delta/stable/dataset_remote.json | 15 ++++ schemas/V_delta/stable/jrclust_clusters.json | 22 ++--- schemas/V_delta/stable/probe_location.json | 34 +++++--- schemas/V_delta/stable/stimulus_response.json | 21 ++++- .../stable/stimulus_response_scalar.json | 55 +++++------- schemas/V_delta/stable/treatment.json | 29 +++++-- 11 files changed, 231 insertions(+), 115 deletions(-) diff --git a/schemas/V_delta/stable/app.json b/schemas/V_delta/stable/app.json index f7f06bb..2780404 100644 --- a/schemas/V_delta/stable/app.json +++ b/schemas/V_delta/stable/app.json @@ -17,7 +17,7 @@ "type": "char", "blank_value": "", "default_value": "ndi.app", - "mustBeNonEmpty": true, + "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, diff --git a/schemas/V_delta/stable/control_stimulus_ids.json b/schemas/V_delta/stable/control_stimulus_ids.json index 85f8a74..5f54048 100644 --- a/schemas/V_delta/stable/control_stimulus_ids.json +++ b/schemas/V_delta/stable/control_stimulus_ids.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "app" } ], "maturity_level": "stable" @@ -12,38 +15,91 @@ "depends_on": [ { "name": "stimulus_presentation_id", - "mustBeNonEmpty": true, - "documentation": "The document ID of the stimulus presentation these control IDs describe.", + "mustBeNonEmpty": false, + "documentation": "Optional pointer to the stimulus_presentation document these control IDs annotate.", "must_refer_to_document_class": "" } ], "file": [], "fields": [ { - "name": "control_ids", - "type": "char", - "blank_value": "", - "default_value": "", + "name": "control_stimulus_ids", + "type": "matrix", + "blank_value": [], + "default_value": [], "mustBeNonEmpty": false, - "mustBeScalar": true, + "mustBeScalar": false, "mustNotHaveNaN": false, "queryable": false, "ontology": null, - "documentation": "Comma-separated list of control stimulus identifiers (e.g., 'blank,gray_screen'). These identify which stimulus conditions serve as controls.", + "documentation": "Integer identifier(s) of the stimulus condition(s) that serve as the control reference for the linked stimulus_presentation. v1 sometimes ships NaN when no control was recorded; sometimes a scalar; sometimes an array.", "constraints": {} }, { - "name": "num_control_ids", - "type": "integer", - "blank_value": 0, - "default_value": 0, + "name": "control_stimulus_id_method", + "type": "structure", + "blank_value": { + "method": "", + "controlid": "", + "controlid_value": 0 + }, + "default_value": { + "method": "", + "controlid": "", + "controlid_value": 0 + }, "mustBeNonEmpty": false, "mustBeScalar": true, - "mustNotHaveNaN": true, + "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The number of unique control stimulus identifiers.", - "constraints": {} + "documentation": "How the control_stimulus_ids were derived. v1 uniformly ships a 3-field struct: method (e.g., 'pseudorandom'), controlid (e.g., 'isblank'), and controlid_value (numeric tag, e.g., 1).", + "constraints": {}, + "fields": [ + { + "name": "method", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Name of the algorithm used to identify which stimuli are controls (e.g., 'pseudorandom', 'explicit_list').", + "constraints": { + "maxLength": 64 + } + }, + { + "name": "controlid", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Name of the per-stimulus key that flags control trials (e.g., 'isblank').", + "constraints": { + "maxLength": 64 + } + }, + { + "name": "controlid_value", + "type": "double", + "blank_value": 0, + "default_value": 0, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Value of `controlid` that marks a trial as a control (e.g., 1).", + "constraints": {} + } + ] } ] } diff --git a/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json b/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json index 5de9d0c..c6e4762 100644 --- a/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json +++ b/schemas/V_delta/stable/daqmetadatareader_epochdata_ingested.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "epochid" } ], "maturity_level": "stable" @@ -15,12 +18,6 @@ "mustBeNonEmpty": true, "documentation": "The document ID of the DAQ metadata reader whose epoch data has been ingested.", "must_refer_to_document_class": "" - }, - { - "name": "epochid", - "mustBeNonEmpty": true, - "documentation": "The document ID of the epoch whose data was ingested.", - "must_refer_to_document_class": "" } ], "file": [], diff --git a/schemas/V_delta/stable/daqreader_epochdata_ingested.json b/schemas/V_delta/stable/daqreader_epochdata_ingested.json index 5166677..21ba839 100644 --- a/schemas/V_delta/stable/daqreader_epochdata_ingested.json +++ b/schemas/V_delta/stable/daqreader_epochdata_ingested.json @@ -26,19 +26,51 @@ "file": [], "fields": [ { - "name": "ingestion_status", - "type": "char", - "blank_value": "", - "default_value": "complete", - "mustBeNonEmpty": true, + "name": "epochtable", + "type": "structure", + "blank_value": { + "epochclock": [], + "t0_t1": [] + }, + "default_value": { + "epochclock": [], + "t0_t1": [] + }, + "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, - "queryable": true, + "queryable": false, "ontology": null, - "documentation": "The status of the ingestion ('complete', 'partial', 'failed').", - "constraints": { - "maxLength": 32 - } + "documentation": "Captured epoch-table snapshot for the ingested epoch. v1 ships {epochclock: [], t0_t1: [t0, t1]} — clock identifier(s) and the corresponding start/stop time pair, both in the units of that clock.", + "constraints": {}, + "fields": [ + { + "name": "epochclock", + "type": "string", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Cell array of clock identifier strings (e.g., {'dev_local_time'}) that index the t0_t1 row(s).", + "constraints": {} + }, + { + "name": "t0_t1", + "type": "matrix", + "blank_value": [], + "default_value": [], + "mustBeNonEmpty": false, + "mustBeScalar": false, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Two-element [t0, t1] start/stop time pair in the units of the matching epochclock entry.", + "constraints": {} + } + ] } ] } diff --git a/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json b/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json index 00a33ad..e441450 100644 --- a/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json +++ b/schemas/V_delta/stable/daqreader_mfdaq_epochdata_ingested.json @@ -5,24 +5,17 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "daqreader_epochdata_ingested" + }, + { + "class_name": "epochid" } ], "maturity_level": "stable" }, - "depends_on": [ - { - "name": "daqreader_id", - "mustBeNonEmpty": true, - "documentation": "The document ID of the MFDAQ reader whose epoch data has been ingested.", - "must_refer_to_document_class": "" - }, - { - "name": "epochid", - "mustBeNonEmpty": true, - "documentation": "The document ID of the epoch whose data was ingested.", - "must_refer_to_document_class": "" - } - ], + "depends_on": [], "file": [], "fields": [ { diff --git a/schemas/V_delta/stable/dataset_remote.json b/schemas/V_delta/stable/dataset_remote.json index 830d8a6..55d7a48 100644 --- a/schemas/V_delta/stable/dataset_remote.json +++ b/schemas/V_delta/stable/dataset_remote.json @@ -41,6 +41,21 @@ "constraints": { "maxLength": 256 } + }, + { + "name": "organization_id", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Organization identifier at the remote service (e.g., the lab or owning group whose namespace the dataset lives under).", + "constraints": { + "maxLength": 256 + } } ] } diff --git a/schemas/V_delta/stable/jrclust_clusters.json b/schemas/V_delta/stable/jrclust_clusters.json index 7eb7597..cdba834 100644 --- a/schemas/V_delta/stable/jrclust_clusters.json +++ b/schemas/V_delta/stable/jrclust_clusters.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "app" } ], "maturity_level": "stable" @@ -25,20 +28,7 @@ ], "fields": [ { - "name": "num_clusters", - "type": "integer", - "blank_value": 0, - "default_value": 0, - "mustBeNonEmpty": false, - "mustBeScalar": true, - "mustNotHaveNaN": true, - "queryable": true, - "ontology": null, - "documentation": "The number of clusters found by JRCLUST.", - "constraints": {} - }, - { - "name": "jrclust_version", + "name": "res_mat_md5_checksum", "type": "char", "blank_value": "", "default_value": "", @@ -47,9 +37,9 @@ "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The version of JRCLUST used for sorting.", + "documentation": "MD5 checksum of the JRCLUST *_res.mat results file, recorded at ingestion so downstream consumers can detect resort/reanalysis.", "constraints": { - "maxLength": 32 + "maxLength": 64 } } ] diff --git a/schemas/V_delta/stable/probe_location.json b/schemas/V_delta/stable/probe_location.json index e6c222c..478b7e5 100644 --- a/schemas/V_delta/stable/probe_location.json +++ b/schemas/V_delta/stable/probe_location.json @@ -20,20 +20,34 @@ "file": [], "fields": [ { - "name": "location", - "type": "ontology_term", - "blank_value": null, - "default_value": null, + "name": "ontology_name", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, - "ontology": { - "node": "schema:location", - "name": "location" - }, - "documentation": "Anatomical or functional location where the probe is sampling, as an ontology term (e.g., UBERON:0002436 'primary visual cortex').", - "constraints": {} + "ontology": null, + "documentation": "Ontology identifier of the probe's anatomical/functional location (e.g., 'UBERON:0002436', 'UBERON:0001880'). v1 ships this as a flat char field alongside `name`.", + "constraints": { + "maxLength": 128 + } + }, + { + "name": "name", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Human-readable name for the location (e.g., 'primary visual cortex', 'bed nucleus of stria terminalis (BNST)'). Paired with `ontology_name`.", + "constraints": { + "maxLength": 256 + } } ] } diff --git a/schemas/V_delta/stable/stimulus_response.json b/schemas/V_delta/stable/stimulus_response.json index 12644eb..530e8ad 100644 --- a/schemas/V_delta/stable/stimulus_response.json +++ b/schemas/V_delta/stable/stimulus_response.json @@ -31,7 +31,7 @@ ], "fields": [ { - "name": "response_type", + "name": "stimulator_epochid", "type": "char", "blank_value": "", "default_value": "", @@ -40,9 +40,24 @@ "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The type of response measured (e.g., 'spikes', 'calcium', 'voltage').", + "documentation": "Epoch identifier on the stimulator element whose presentation produced this response (e.g., 't00002'). Mirrors the v1 stimulus_response field.", "constraints": { - "maxLength": 128 + "maxLength": 64 + } + }, + { + "name": "element_epochid", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Epoch identifier on the responding element (the element_id this document depends on) that aligns with the stimulator epoch above.", + "constraints": { + "maxLength": 64 } } ] diff --git a/schemas/V_delta/stable/stimulus_response_scalar.json b/schemas/V_delta/stable/stimulus_response_scalar.json index baae2cf..9a3d638 100644 --- a/schemas/V_delta/stable/stimulus_response_scalar.json +++ b/schemas/V_delta/stable/stimulus_response_scalar.json @@ -5,6 +5,9 @@ "superclasses": [ { "class_name": "base" + }, + { + "class_name": "stimulus_response" } ], "maturity_level": "stable" @@ -12,45 +15,15 @@ "depends_on": [ { "name": "stimulus_response_id", - "mustBeNonEmpty": true, - "documentation": "The document ID of the stimulus_response this scalar summary belongs to.", + "mustBeNonEmpty": false, + "documentation": "Optional pointer to the underlying stimulus_response document.", "must_refer_to_document_class": "" } ], "file": [], "fields": [ { - "name": "stimulus_values", - "type": "matrix", - "blank_value": [], - "default_value": [], - "mustBeNonEmpty": false, - "mustBeScalar": false, - "mustNotHaveNaN": false, - "queryable": false, - "ontology": null, - "documentation": "Array of independent stimulus parameter values, one per stimulus condition. Must be the same length as response_values.", - "constraints": { - "element_type": "double" - } - }, - { - "name": "response_values", - "type": "matrix", - "blank_value": [], - "default_value": [], - "mustBeNonEmpty": false, - "mustBeScalar": false, - "mustNotHaveNaN": false, - "queryable": false, - "ontology": null, - "documentation": "Array of scalar response values in response_units, one per stimulus condition. Must be the same length as stimulus_values.", - "constraints": { - "element_type": "double" - } - }, - { - "name": "response_units", + "name": "response_type", "type": "char", "blank_value": "", "default_value": "", @@ -59,10 +32,24 @@ "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The units of the response values (e.g., 'spikes/s', 'dF/F').", + "documentation": "How the per-trial scalar response was reduced from the source signal (e.g., 'mean', 'peak', 'frequency'). v1 ships this directly on the stimulus_response_scalar block; declared here so per-class queries see it without joining to the parent.", "constraints": { "maxLength": 64 } + }, + { + "name": "responses", + "type": "structure", + "blank_value": {}, + "default_value": {}, + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": false, + "ontology": null, + "documentation": "Per-trial scalar responses. v1 ships this as a struct with sub-fields {stimid (integer), response_real, response_imaginary, control_response_real, control_response_imaginary}; V_delta keeps the open-shape convention so callers can add bookkeeping fields without a schema bump.", + "constraints": {}, + "fields": [] } ] } diff --git a/schemas/V_delta/stable/treatment.json b/schemas/V_delta/stable/treatment.json index 39c912f..fd506f0 100644 --- a/schemas/V_delta/stable/treatment.json +++ b/schemas/V_delta/stable/treatment.json @@ -32,17 +32,34 @@ "file": [], "fields": [ { - "name": "treatment_name", - "type": "ontology_term", - "blank_value": null, - "default_value": null, + "name": "ontology_name", + "type": "char", + "blank_value": "", + "default_value": "", "mustBeNonEmpty": false, "mustBeScalar": true, "mustNotHaveNaN": false, "queryable": true, "ontology": null, - "documentation": "The treatment, identified as an ontology term.", - "constraints": {} + "documentation": "Ontology identifier of the treatment (e.g., 'EMPTY:0000074'). v1 ships this as a flat char field alongside `name`.", + "constraints": { + "maxLength": 128 + } + }, + { + "name": "name", + "type": "char", + "blank_value": "", + "default_value": "", + "mustBeNonEmpty": false, + "mustBeScalar": true, + "mustNotHaveNaN": false, + "queryable": true, + "ontology": null, + "documentation": "Human-readable name for the treatment. Paired with `ontology_name`.", + "constraints": { + "maxLength": 256 + } }, { "name": "numeric_value",