Merge pull request #43 from Waltham-Data-Science/claude/add-symmetry-tests-7IWoH

Add symmetry tests for downloaded ingested datasets

Author: stevevanhooser

.github/workflows/test-symmetry.yml

@@ -60,6 +60,12 @@ jobs:
           python -m pip install --upgrade pip
           python ndi_install.py --dev --no-validate --verbose
 
+      # ── Download test datasets ────────────────────────────────────────
+      - name: Download test datasets
+        run: |
+          curl -L -o /tmp/69a8705aa9ab25373cdc6563.tgz \
+            https://github.com/Waltham-Data-Science/file-passing/raw/refs/heads/main/69a8705aa9ab25373cdc6563.tgz
+
       # ── Stage 1: MATLAB makeArtifacts ──────────────────────────────────
       - name: "Stage 1: MATLAB makeArtifacts"
         uses: matlab-actions/run-command@v2

examples/integration_demo.py

@@ -23,6 +23,7 @@
 # Cofounder's compression library
 try:
     import ndicompress
+
     HAS_COMPRESS = True
 except ImportError:
     HAS_COMPRESS = False
@@ -60,7 +61,7 @@ def demo_full_workflow():
         spike_times = np.random.choice(num_samples, size=20, replace=False)
         for st in spike_times:
             if st + 30 < num_samples:
-                data[st:st+30, ch] += np.sin(np.linspace(0, np.pi, 30)) * 500
+                data[st : st + 30, ch] += np.sin(np.linspace(0, np.pi, 30)) * 500
 
     data = data.astype(np.int16)  # Convert to int16 for ephys
     print(f"   Data shape: {data.shape}")
@@ -90,40 +91,30 @@ def demo_full_workflow():
             # In real usage, we'd load from JSON schema
             # For demo, we create a simple document structure
             doc_props = {
-                'base': {
-                    'id': Ido().id,
-                    'datestamp': timestamp(),
-                    'name': 'ephys_recording_001',
-                    'session_id': ''
-                },
-                'document_class': {
-                    'class_name': 'ndi_document_ephys',
-                    'superclasses': []
+                "base": {
+                    "id": Ido().id,
+                    "datestamp": timestamp(),
+                    "name": "ephys_recording_001",
+                    "session_id": "",
                 },
-                'ephys': {
-                    'num_channels': num_channels,
-                    'sample_rate': sample_rate,
-                    'num_samples': num_samples,
-                    'duration_seconds': num_samples / sample_rate,
-                    'data_type': 'int16',
-                    'compression': 'ndi_compress_ephys'
+                "document_class": {"class_name": "ndi_document_ephys", "superclasses": []},
+                "ephys": {
+                    "num_channels": num_channels,
+                    "sample_rate": sample_rate,
+                    "num_samples": num_samples,
+                    "duration_seconds": num_samples / sample_rate,
+                    "data_type": "int16",
+                    "compression": "ndi_compress_ephys",
                 },
-                'files': {
-                    'file_list': ['ephys_data.nbf_#'],
-                    'file_info': []
-                }
+                "files": {"file_list": ["ephys_data.nbf_#"], "file_info": []},
             }
 
             doc = Document(doc_props)
 
             # === Step 4: Link compressed file to document ===
             print("\n[4] Linking compressed file to document...")
 
-            doc = doc.add_file(
-                name='ephys_data.nbf_1',
-                location=compressed_file,
-                ingest=True
-            )
+            doc = doc.add_file(name="ephys_data.nbf_1", location=compressed_file, ingest=True)
 
             print(f"   Document ID: {doc.id}")
             print(f"   Document class: {doc.doc_class()}")
@@ -133,9 +124,9 @@ def demo_full_workflow():
             print("\n[5] Query demonstration...")
 
             # These queries would work with ndi.database
-            q1 = Query('ephys.num_channels') == 4
-            q2 = Query('ephys.sample_rate') > 20000
-            q3 = Query('base.name').contains('ephys')
+            q1 = Query("ephys.num_channels") == 4
+            q2 = Query("ephys.sample_rate") > 20000
+            q3 = Query("base.name").contains("ephys")
 
             # Combined query
             _q_combined = q1 & q2 & q3
@@ -166,9 +157,9 @@ def demo_full_workflow():
         # Still show document creation without compression
         print("\n[3] Creating ndi.Document (without compression)...")
 
-        doc = Document('base')
-        doc = doc.set_session_id('demo_session')
-        doc = doc.setproperties(**{'base.name': 'ephys_demo'})
+        doc = Document("base")
+        doc = doc.set_session_id("demo_session")
+        doc = doc.setproperties(**{"base.name": "ephys_demo"})
 
         print(f"   Document ID: {doc.id}")
         print(f"   Session ID: {doc.session_id}")
@@ -214,17 +205,19 @@ def demo_document_features():
     print("=" * 60)
 
     # Create a document
-    doc = Document('base')
+    doc = Document("base")
     print(f"\n1. Created document with ID: {doc.id}")
 
     # Set session
-    doc = doc.set_session_id('session_abc123')
+    doc = doc.set_session_id("session_abc123")
     print(f"2. Set session ID: {doc.session_id}")
 
     # Bulk property setting (useful for ephys metadata)
-    doc = doc.setproperties(**{
-        'base.name': 'neural_recording',
-    })
+    doc = doc.setproperties(
+        **{
+            "base.name": "neural_recording",
+        }
+    )
     print("3. Set name via setproperties")
 
     # Document equality (by ID)
@@ -233,9 +226,9 @@ def demo_document_features():
 
     # Static methods for working with document arrays
     docs = [
-        Document('base'),
-        Document('base'),
-        Document('base'),
+        Document("base"),
+        Document("base"),
+        Document("base"),
     ]
 
     newest, idx, ts = Document.find_newest(docs)
@@ -251,6 +244,6 @@ def demo_document_features():
     print("\nAll features work and are ready to integrate with ndicompress!")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     demo_full_workflow()
     demo_document_features()

pythonArtifacts.tar.gz

@@ -60,13 +60,13 @@ def __init__(
 
         # Load from document if provided
         if document is not None:
-            doc_props = getattr(document, "document_properties", document)
-            if hasattr(doc_props, "base") and hasattr(doc_props.base, "id"):
-                self._id = doc_props.base.id
-            if hasattr(doc_props, "daqmetadatareader"):
-                self._tab_separated_file_parameter = getattr(
-                    doc_props.daqmetadatareader, "tab_separated_file_parameter", ""
-                )
+            doc_props = document.document_properties
+            base_id = doc_props.get("base", {}).get("id")
+            if base_id:
+                self._id = base_id
+            dmr = doc_props.get("daqmetadatareader", {})
+            if dmr:
+                self._tab_separated_file_parameter = dmr.get("tab_separated_file_parameter", "")
 
     @property
     def tab_separated_file_parameter(self) -> str:

src/ndi/daq/metadatareader/__init__.py

@@ -568,7 +568,7 @@ def getchannelsepoch_ingested(
         See also: getchannelsepoch
         """
         doc = self.getingesteddocument(epochfiles, session)
-        et = doc.document_properties.daqreader_epochdata_ingested.epochtable
+        et = doc.document_properties["daqreader_epochdata_ingested"]["epochtable"]
 
         channels_raw = et.get("channels", [])
         channels = []
@@ -618,7 +618,7 @@ def readchannels_epochsamples_ingested(
         See also: readchannels_epochsamples
         """
         doc = self.getingesteddocument(epochfiles, session)
-        et = doc.document_properties.daqreader_epochdata_ingested.epochtable
+        et = doc.document_properties["daqreader_epochdata_ingested"]["epochtable"]
 
         # Normalize inputs
         if isinstance(channel, int):

src/ndi/daq/mfdaq.py

@@ -56,9 +56,10 @@ def __init__(
 
         # Load from document if provided
         if document is not None:
-            doc_props = getattr(document, "document_properties", document)
-            if hasattr(doc_props, "base") and hasattr(doc_props.base, "id"):
-                self._id = doc_props.base.id
+            doc_props = document.document_properties
+            base_id = doc_props.get("base", {}).get("id")
+            if base_id:
+                self._id = base_id
 
     def epochclock(
         self,
@@ -160,8 +161,8 @@ def ingested2epochs_t0t1_epochclock(
 
         for doc in d_ingested:
             props = doc.document_properties
-            epochid = props.epochid.epochid
-            et = props.daqreader_epochdata_ingested.epochtable
+            epochid = props["epochid"]["epochid"]
+            et = props["daqreader_epochdata_ingested"]["epochtable"]
 
             # Extract epoch clock
             ec_list = []
@@ -201,7 +202,8 @@ def epochclock_ingested(
         See also: epochclock, ndi_time_clocktype
         """
         doc = self.getingesteddocument(epochfiles, session)
-        et = doc.document_properties.daqreader_epochdata_ingested.epochtable
+        props = doc.document_properties
+        et = props["daqreader_epochdata_ingested"]["epochtable"]
 
         ec_list = []
         for ec_str in et.get("epochclock", []):
@@ -230,7 +232,8 @@ def t0_t1_ingested(
         See also: t0_t1, epochclock_ingested
         """
         doc = self.getingesteddocument(epochfiles, session)
-        et = doc.document_properties.daqreader_epochdata_ingested.epochtable
+        props = doc.document_properties
+        et = props["daqreader_epochdata_ingested"]["epochtable"]
 
         t0t1_raw = et.get("t0_t1", [])
         if not isinstance(t0t1_raw, list):
@@ -288,9 +291,24 @@ def ingest_epochfiles(
         ec_strings = [c.value if isinstance(c, ndi_time_clocktype) else str(c) for c in ec]
         t0t1 = self.t0_t1(epochfiles)
 
+        # Convert NaN values to None so json.dumps produces null instead of
+        # non-standard NaN that MATLAB's jsondecode cannot parse.
+        import math
+
+        sanitized_t0t1 = []
+        for pair in t0t1:
+            if isinstance(pair, (list, tuple)):
+                sanitized_t0t1.append(
+                    [None if (isinstance(v, float) and math.isnan(v)) else v for v in pair]
+                )
+            else:
+                sanitized_t0t1.append(
+                    None if (isinstance(pair, float) and math.isnan(pair)) else pair
+                )
+
         epochtable = {
             "epochclock": ec_strings,
-            "t0_t1": t0t1,
+            "t0_t1": sanitized_t0t1,
         }
 
         doc = ndi_document(

src/ndi/daq/reader_base.py

@@ -42,6 +42,17 @@ def _serialize_t0_t1(t0_t1: Any) -> list:
     return t0_t1
 
 
+def _serialize_single_epochprobemap(epm: Any) -> dict[str, Any]:
+    """Convert a single epochprobemap object to a JSON-compatible dict."""
+    if isinstance(epm, dict):
+        return epm
+    if hasattr(epm, "to_dict"):
+        return epm.to_dict()
+    if hasattr(epm, "__dict__"):
+        return {k: v for k, v in epm.__dict__.items() if not k.startswith("_")}
+    return epm
+
+
 def _serialize_epochnode(node: dict[str, Any]) -> None:
     """Normalize epoch node dict in-place to MATLAB-compatible JSON format."""
     # epoch_clock: list of ClockType -> single dict (MATLAB unwraps single)
@@ -68,12 +79,12 @@ def _serialize_epochnode(node: dict[str, Any]) -> None:
         if isinstance(ue_t, list):
             ue["t0_t1"] = [_serialize_t0_t1(t) for t in ue_t]
 
-    # epochprobemap: if it's a list, handle; if it has a to_dict method, use it
+    # epochprobemap: serialize to JSON-compatible format
     epm = node.get("epochprobemap")
-    if epm is not None and hasattr(epm, "to_dict"):
-        node["epochprobemap"] = epm.to_dict()
-    elif epm is not None and hasattr(epm, "__dict__") and not isinstance(epm, dict):
-        node["epochprobemap"] = {k: v for k, v in epm.__dict__.items() if not k.startswith("_")}
+    if isinstance(epm, list):
+        node["epochprobemap"] = [_serialize_single_epochprobemap(item) for item in epm]
+    elif epm is not None and not isinstance(epm, (dict, str)):
+        node["epochprobemap"] = _serialize_single_epochprobemap(epm)
 
 
 class ndi_daq_system(ndi_ido):
@@ -460,8 +471,10 @@ def getprobes(self) -> list[dict[str, Any]]:
             - name: ndi_probe name
             - reference: ndi_probe reference
             - type: ndi_probe type
-            - subject_id: ndi_subject identifier
+            - subject_id: ndi_subject document ID
         """
+        from ..subject import ndi_subject
+
         et = self.epochtable()
         probes = []
         seen = set()
@@ -481,12 +494,24 @@ def getprobes(self) -> list[dict[str, Any]]:
                         key = (item.name, item.reference, item.type)
                         if key not in seen:
                             seen.add(key)
+                            # Look up the subject document ID from the
+                            # subjectstring (e.g. "anteater27@nosuchlab.org").
+                            # MATLAB does the same lookup and stores the
+                            # document ID, not the local_identifier string.
+                            subjectstring = getattr(item, "subjectstring", "")
+                            sid = ""
+                            if subjectstring and self.session is not None:
+                                _, doc_id = ndi_subject.does_subjectstring_match_session_document(
+                                    self.session, subjectstring
+                                )
+                                if doc_id is not None:
+                                    sid = doc_id
                             probes.append(
                                 {
                                     "name": item.name,
                                     "reference": item.reference,
                                     "type": item.type,
-                                    "subject_id": getattr(item, "subjectstring", ""),
+                                    "subject_id": sid,
                                 }
                             )
 
@@ -600,7 +625,7 @@ def ingest(self) -> tuple[bool, list[Any]]:
 
         # Set session IDs and add to database
         if self.session is not None:
-            session_id = self.session.id
+            session_id = self.session.id()
             for doc in docs:
                 doc.set_session_id(session_id)
             self.session.database_add(docs)
@@ -746,7 +771,7 @@ def newdocument(self) -> list[Any]:
         )
 
         if self.session is not None:
-            sys_doc.set_session_id(self.session.id)
+            sys_doc.set_session_id(self.session.id())
 
         if self._filenavigator is not None:
             sys_doc.set_dependency_value("filenavigator_id", self._filenavigator.id)
@@ -775,7 +800,7 @@ def searchquery(self) -> Any:
         if self._name:
             q = q & (ndi_query("base.name") == self._name)
         if self.session is not None:
-            q = q & (ndi_query("base.session_id") == self.session.id)
+            q = q & (ndi_query("base.session_id") == self.session.id())
 
         return q