[did2 #3] Flip NDI_DID2_NORMALIZE_ON_READ — always normalise on read

src/ndi/+ndi/+calc/+example/simple.m

@@ -96,16 +96,16 @@
             % 1. Create mock documents if they don't exist
 
             % Check/Create Doc for Test 1 (Value 5)
-            q1 = ndi.query('','isa','demoNDIMock','');
-            q2 = ndi.query('demoNDI.value','exact_number', 5, '');
+            q1 = ndi.query('','isa','demo_ndi_mock','');
+            q2 = ndi.query('demo_ndi.value','exact_number', 5, '');
             q_test1 = q1 & q2;
             docs_test1 = ndi_calculator_obj.session.database_search(q_test1);
 
             if isempty(docs_test1)
-                mock_doc1_struct.demoNDI.value = 5;
-                % demoNDIMock requires a file because it inherits from demoNDI
-                mock_doc1 = ndi.document('demoNDIMock', 'demoNDI', mock_doc1_struct.demoNDI) + ndi_calculator_obj.session.newdocument();
-                % We need to add a dummy file because demoNDI schema requires 'filename1.ext'
+                mock_doc1_struct.demo_ndi.value = 5;
+                % demo_ndi_mock requires a file because it inherits from demo_ndi
+                mock_doc1 = ndi.document('demo_ndi_mock', 'demo_ndi', mock_doc1_struct.demo_ndi) + ndi_calculator_obj.session.newdocument();
+                % We need to add a dummy file because demo_ndi schema requires 'filename1.ext'
                 % Ideally we create a dummy file on disk.
                 fname1 = [ndi_calculator_obj.session.path() filesep 'test1_dummy.txt'];
                 vlt.file.str2text(fname1, 'dummy content');
@@ -117,13 +117,13 @@
             end
 
             % Check/Create Doc for Test 2 (Value 10)
-            q3 = ndi.query('demoNDI.value','exact_number', 10, '');
+            q3 = ndi.query('demo_ndi.value','exact_number', 10, '');
             q_test2 = q1 & q3;
             docs_test2 = ndi_calculator_obj.session.database_search(q_test2);
 
             if isempty(docs_test2)
-                mock_doc2_struct.demoNDI.value = 10;
-                mock_doc2 = ndi.document('demoNDIMock', 'demoNDI', mock_doc2_struct.demoNDI) + ndi_calculator_obj.session.newdocument();;
+                mock_doc2_struct.demo_ndi.value = 10;
+                mock_doc2 = ndi.document('demo_ndi_mock', 'demo_ndi', mock_doc2_struct.demo_ndi) + ndi_calculator_obj.session.newdocument();;
                 fname2 = [ndi_calculator_obj.session.path() filesep 'test2_dummy.txt'];
                 vlt.file.str2text(fname2, 'dummy content');
                 mock_doc2 = mock_doc2.add_file('filename1.ext', fname2);
@@ -154,7 +154,7 @@
                 end
 
                 % Setup Query for this specific test
-                q_val = ndi.query('demoNDI.value', 'exact_number', target_value, '');
+                q_val = ndi.query('demo_ndi.value', 'exact_number', target_value, '');
                 q_combined = q1 & q_val;
 
                 search_params = struct('input_parameters', struct('answer', target_value), ...

src/ndi/+ndi/+compat/fieldAliases.m

@@ -67,12 +67,6 @@
             {'ontology_label.ontology_name', 'ontology_label.label_id'}, ...
             {@i_labelNodeToVDelta, @i_labelNodeToLegacy}; ...
         'ontology_label.term.name', 'ontology_label.label', []; ...
-        ...
-        % daqmetadatareader: V_delta renamed the MATLAB-class storage
-        % field. did2 migrator copies the v1 value verbatim; the alias
-        % keeps legacy reader/writer code working at runtime.
-        'daqmetadatareader.reader_class', ...
-            'daqmetadatareader.ndi_daqmetadatareader_class', []; ...
     };
 end
 

src/ndi/+ndi/+database/+fun/ndi_document2ndi_object.m

@@ -37,41 +37,7 @@
         error(['NDI_DOCUMENT_OBJ does not have a ''' obj_parent_string  ''' field.']);
     else
         obj_struct = getfield(ndi_document_obj.document_properties, obj_parent_string);
-        obj_string = resolveReconstructorClass(obj_parent_string, obj_struct);
+        obj_string = getfield(obj_struct,['ndi_' obj_parent_string '_class']);
     end
 
     o = eval([obj_string '(ndi_session_obj, ndi_document_obj);']);
-
-function obj_string = resolveReconstructorClass(obj_parent_string, obj_struct)
-    % resolveReconstructorClass - determine the MATLAB class to
-    % `eval()` for the document's reconstruction.
-    %
-    % did_v1 stored this class name on the body under
-    % `<parent>.ndi_<parent>_class`. Most V_delta migrators pass
-    % the field through unchanged, but a few drop it on the
-    % grounds that the stored value was a constant across every
-    % instance of the class (so the migration loses no
-    % information). For those classes we know the class name from
-    % the document class itself and resolve it here.
-
-    % Override map: V_delta class name -> MATLAB constructor class.
-    % daqreader_ndr always reconstructs as ndi.daq.reader.mfdaq.ndr
-    % (verified: every v1 daqreader_ndr blank stored the same value;
-    % did-schema#50 / did-matlab#135 audit).
-    overrides = struct( ...
-        'daqreader_ndr', 'ndi.daq.reader.mfdaq.ndr');
-
-    legacyField = ['ndi_' obj_parent_string '_class'];
-    if isfield(obj_struct, legacyField) && ~isempty(obj_struct.(legacyField))
-        obj_string = obj_struct.(legacyField);
-        return;
-    end
-    if isfield(overrides, obj_parent_string)
-        obj_string = overrides.(obj_parent_string);
-        return;
-    end
-    error('NDI:database:fun:noReconstructorClass', ...
-        ['Cannot determine MATLAB reconstructor class for ' ...
-         'document class "%s": neither the legacy field "%s" ' ...
-         'nor a known override is available.'], ...
-        obj_parent_string, legacyField);

src/ndi/+ndi/+database/+internal/applyReadNormalization.m

@@ -3,32 +3,27 @@
 %
 %   NDIDOCUMENTOBJ = ndi.database.internal.applyReadNormalization(RAWDOC)
 %   wraps a body read from a concrete ndi.database backend in an
-%   ndi.document. When the env var NDI_DID2_NORMALIZE_ON_READ is set to
-%   '1', RAWDOC is first routed through did2.convert.v1_to_v2 so v1
+%   ndi.document. RAWDOC is routed through did2.convert.v1_to_v2 so v1
 %   bodies are returned as V_delta-shaped documents. V_delta bodies
 %   short-circuit the converter's idempotency check, so re-reads of
 %   already-V_delta documents stay cheap.
 %
-%   The env-var gate is OFF by default so this PR can land ahead of the
-%   companion work that the issue body lists as dependencies:
-%     - "Issue 1": NDI schemas converted to V_delta (every blank doc
-%       under src/ndi/ndi_common/database_documents/ is still v1, and
-%       ndi.document.readblankdefinition reads from there).
-%     - #779: in-memory compat shim that injects legacy aliases on
-%       ndi.document read, so existing callers (e.g.,
-%       ndi.database.fun.ndi_document2ndi_object,
-%       ndi.daq.metadatareader) keep finding v1 field names like
-%       ndi_daqmetadatareader_class after V_delta normalisation
-%       renames them to reader_class.
-%   With the gate OFF the wiring + tests in this file are dormant; flip
-%   to ON via setenv('NDI_DID2_NORMALIZE_ON_READ','1') once #779 / issue
-%   1 ship to activate normalisation across every ndi.database backend.
+%   The in-memory ndi.document then carries the did_v1 legacy alias
+%   paths injected by ndi.compat.augmentRead (issue #779) plus
+%   canonical `depends_on.document_id` entries via
+%   ndi.compat.normalizeDependsOn (issue #801). Callers that still
+%   read legacy field names (e.g.,
+%   ndi.database.fun.ndi_document2ndi_object,
+%   ndi.daq.metadatareader, customer code reading
+%   document_properties.probe_location.ontology_name) keep working
+%   even though storage normalised to V_delta. The write-side mirror
+%   lives in ndi.compat.reconcileWrite (issue #780); query-side path
+%   translation lives in ndi.compat.translateQueryPaths (issue #781).
 %
 %   Concrete ndi.database subclasses call this helper from their
 %   do_read / do_search implementations so the abstract ndi.database
-%   API stays byte-identical regardless of the gate state: callers
-%   above the abstraction (session, dataset, queries) only ever see
-%   ndi.document objects.
+%   API stays byte-identical: callers above the abstraction (session,
+%   dataset, queries) only ever see ndi.document objects.
 %
 %   RAWDOC may be:
 %       - a struct                (the body itself),
@@ -40,12 +35,14 @@
 %   Errors:
 %       NDI:database:normalizeBadInput  - RAWDOC is not a recognised
 %                                         document/body type.
-%       NDI:database:normalizeFailed    - the gate is ON and
-%                                         did2.convert.v1_to_v2
+%       NDI:database:normalizeFailed    - did2.convert.v1_to_v2
 %                                         quarantined the body so no
 %                                         migrated document was produced.
 %
-%   See also: did2.convert.v1_to_v2, ndi.database,
+%   See also: did2.convert.v1_to_v2, ndi.compat.augmentRead,
+%             ndi.compat.normalizeDependsOn,
+%             ndi.compat.reconcileWrite, ndi.compat.translateQueryPaths,
+%             ndi.database,
 %             ndi.database.implementations.database.didsqlite,
 %             ndi.database.implementations.database.matlabdumbjsondb2.
 
@@ -70,14 +67,6 @@
              '(got "%s").'], class(rawDoc));
     end
 
-    if ~normalizationGateOn()
-        % Gate OFF: preserve the pre-#776 behaviour (just wrap the
-        % body) so the rest of the codebase keeps finding v1 field
-        % names until issue 1 / #779 land.
-        ndiDocumentObj = ndi.document(body);
-        return;
-    end
-
     % Validate=false on the read path: the body was validated when it
     % was written, and re-validating every read for every doc burns
     % time on production workloads. The migrate command and the write
@@ -99,14 +88,3 @@
 
     ndiDocumentObj = ndi.document(result.migrated{1}.toStruct());
 end
-
-function tf = normalizationGateOn()
-% Read NDI_DID2_NORMALIZE_ON_READ and treat '1', 'true', 'yes', 'on' as
-% ON (case-insensitive). Anything else (including unset) is OFF.
-raw = getenv('NDI_DID2_NORMALIZE_ON_READ');
-if isempty(raw)
-    tf = false;
-    return;
-end
-tf = any(strcmpi(strtrim(raw), {'1', 'true', 'yes', 'on'}));
-end

src/ndi/+ndi/+migrate/cloud.m

@@ -8,7 +8,7 @@
 %   did2.convert.v1_to_v2 (idempotent — already-V_delta docs are
 %   skipped cheaply), and the surviving bodies are pushed back via the
 %   existing bulk-upload endpoint. Because each successfully written
-%   doc carries `base.schema_version: 'V_delta'`, an interrupted run
+%   doc carries `document_class.schema_version: 'V_delta'`, an interrupted run
 %   can resume by re-running this command — already-migrated docs
 %   short-circuit through the conversion.
 %

src/ndi/+ndi/+test/+database/test_ndi_document.m

@@ -23,16 +23,16 @@ function test_ndi_document(dirname)
 
     % if we ran the demo before, delete the entry
 
-    doc = E.database_search(ndi.query('','isa','demoNDI',''));
+    doc = E.database_search(ndi.query('','isa','demo_ndi',''));
     if ~isempty(doc)
         for i=1:numel(doc)
             E.database_rm(id(doc{i}));
         end
     end
 
-    doc = E.newdocument('demoNDI',...
+    doc = E.newdocument('demo_ndi',...
         'base.name','Demo document',...
-        'demoNDI.value', 5);
+        'demo_ndi.value', 5);
 
     % add a binary file
 
@@ -54,15 +54,15 @@ function test_ndi_document(dirname)
 
     % now read the object back
 
-    doc = E.database_search(ndi.query('demoNDI.value','exact_number',5,''));
+    doc = E.database_search(ndi.query('demo_ndi.value','exact_number',5,''));
     if numel(doc)~=1
-        error(['Found <1 or >1 document with demoNDI.value of 5; this means there is a database problem.']);
+        error(['Found <1 or >1 document with demo_ndi.value of 5; this means there is a database problem.']);
     end
     doc = doc{1}, % should be only one match
 
-    doc = E.database_search(ndi.query('','isa','demoNDI',''));
+    doc = E.database_search(ndi.query('','isa','demo_ndi',''));
     if numel(doc)~=1
-        error(['Found <1 or >1 document of type demoNDI; this means there is a database problem.']);
+        error(['Found <1 or >1 document of type demo_ndi; this means there is a database problem.']);
     end
     doc = doc{1}, % should be only one match
 
@@ -78,7 +78,7 @@ function test_ndi_document(dirname)
 
     % remove the document
 
-    doc = E.database_search(ndi.query('','isa','demoNDI',''));
+    doc = E.database_search(ndi.query('','isa','demo_ndi',''));
     if ~isempty(doc)
         for i=1:numel(doc)
             E.database_rm(doc{i}.id());

src/ndi/+ndi/element.m

@@ -405,12 +405,15 @@
                     newet.epoch_number = i;
                     newet.epoch_id = potential_epochdocs{i}.document_properties.epochid.epochid;
                     newet.epochprobemap = '';
-                    clock_types = strtrim(split(potential_epochdocs{i}.document_properties.element_epoch.epoch_clock,','));
+                    % V_delta element_epoch.clocks: array-of-records
+                    % with fields {name, t0, t1}. Replaces the v1
+                    % parallel (epoch_clock CSV + t0_t1 matrix) shape.
+                    clocks_array = potential_epochdocs{i}.document_properties.element_epoch.clocks;
                     ec = {};
                     t0_t1 = {};
-                    for k=1:numel(clock_types)
-                        ec{k} = ndi.time.clocktype(clock_types{k});
-                        t0_t1{k} = vlt.data.rowvec(potential_epochdocs{i}.document_properties.element_epoch.t0_t1(:,k));
+                    for k=1:numel(clocks_array)
+                        ec{k} = ndi.time.clocktype(char(clocks_array(k).name));
+                        t0_t1{k} = vlt.data.rowvec([clocks_array(k).t0, clocks_array(k).t1]);
                     end
                     newet.epoch_clock = ec;
                     newet.t0_t1 = t0_t1;

src/ndi/ndi_common/database_documents/demoNDI.json → src/ndi/ndi_common/database_documents/demo_ndi.json

@@ -1,9 +1,9 @@
 {
 	"document_class": {
-		"definition":						"$NDIDOCUMENTPATH\/demoNDI.json",
-		"validation":						"$NDISCHEMAPATH\/demoNDI_schema.json",
-		"class_name":						"demoNDI",
-		"property_list_name":					"demoNDI",
+		"definition":						"$NDIDOCUMENTPATH\/demo_ndi.json",
+		"validation":						"$NDISCHEMAPATH\/demo_ndi_schema.json",
+		"class_name":						"demo_ndi",
+		"property_list_name":					"demo_ndi",
 		"class_version":					1,
 		"superclasses": [
 			{ "definition":					"$NDIDOCUMENTPATH\/base.json" }
@@ -14,7 +14,7 @@
 			"filename1.ext"
 		]
 	},
-	"demoNDI": {
+	"demo_ndi": {
 		"value":						""
 	}
 }

src/ndi/ndi_common/database_documents/mock/demo_ndi_mock.json

@@ -0,0 +1,15 @@
+{
+	"document_class": {
+		"definition":						"$NDIDOCUMENTPATH\/mock\/demo_ndi_mock.json",
+		"validation":						"$NDISCHEMAPATH\/mock\/demo_ndi_mock.json",
+		"class_name":						"demo_ndi_mock",
+		"property_list_name":					"demo_ndi_mock",
+		"class_version":					1,
+		"superclasses": [
+			{ "definition":					"$NDIDOCUMENTPATH\/mock.json" },
+			{ "definition":					"$NDIDOCUMENTPATH\/demo_ndi.json" }
+		]
+	},
+	"demo_ndi_mock": {
+	}
+}

src/ndi/ndi_common/resources/ndiDocumentAttributes.json

@@ -54,7 +54,7 @@
       "Data acqusiiton metadata"
     ]
   },
-  "demoNDI": {
+  "demo_ndi": {
     "attributes": [
       "Calculation"
     ]