Add ndi.util.datasetSummary and compareDatasetSummary utilities

claude · claude · commit bba4b0752ad6 · 2026-03-21T16:54:20.000Z
Extract dataset summary logic from symmetry tests into public ndi.util functions (mirroring MATLAB's ndi.util.datasetSummary and ndi.util.compareDatasetSummary). Simplify symmetry tests to use the new utilities instead of inline summary building and comparison. Add 14 unit tests covering both functions. https://claude.ai/code/session_01EctVW1VcbY2LzAdfZrGBB5
diff --git a/src/ndi/util/__init__.py b/src/ndi/util/__init__.py
@@ -14,7 +14,9 @@
 """
 
 from .classname import ndi_matlab_classname, ndi_python_classname
+from .compare_dataset_summary import compareDatasetSummary
 from .compare_session_summary import compareSessionSummary
+from .dataset_summary import datasetSummary
 from .datestamp2datetime import datestamp2datetime
 from .downsampleTimeseries import downsampleTimeseries
 from .getHexDiffFromFileObj import getHexDiffFromFileObj
@@ -28,7 +30,9 @@
 __all__ = [
     "ndi_matlab_classname",
     "ndi_python_classname",
+    "compareDatasetSummary",
     "compareSessionSummary",
+    "datasetSummary",
     "datestamp2datetime",
     "downsampleTimeseries",
     "getHexDiffFromFileObj",
diff --git a/src/ndi/util/compare_dataset_summary.py b/src/ndi/util/compare_dataset_summary.py
@@ -0,0 +1,119 @@
+"""Compare two dataset summaries and return a report of differences.
+
+MATLAB equivalent: ``ndi.util.compareDatasetSummary``
+
+Compares two summary dicts (as produced by :func:`datasetSummary`) and
+returns a list of human-readable difference strings.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .compare_session_summary import compareSessionSummary
+
+
+def compareDatasetSummary(
+    summary1: dict[str, Any],
+    summary2: dict[str, Any],
+    *,
+    excludeFiles: list[str] | None = None,
+    excludeFields: list[str] | None = None,
+) -> list[str]:
+    """Compare two dataset summaries and return a report.
+
+    MATLAB equivalent: ``ndi.util.compareDatasetSummary(s1, s2, ...)``
+
+    Args:
+        summary1: First dataset summary dict.
+        summary2: Second dataset summary dict.
+        excludeFiles: Filenames to ignore when comparing file lists
+            within session summaries.
+        excludeFields: Field names to skip entirely during comparison.
+
+    Returns:
+        List of difference strings. Empty list means summaries match.
+    """
+    if excludeFiles is None:
+        excludeFiles = []
+    if excludeFields is None:
+        excludeFields = []
+
+    report: list[str] = []
+
+    # 1. Compare numSessions
+    if "numSessions" not in excludeFields:
+        n1 = summary1.get("numSessions", 0)
+        n2 = summary2.get("numSessions", 0)
+        if n1 != n2:
+            report.append(
+                f"numSessions differs: {n1} vs {n2}"
+            )
+
+    # 2. Compare references
+    if "references" not in excludeFields:
+        refs1 = sorted(summary1.get("references", []))
+        refs2 = sorted(summary2.get("references", []))
+        if refs1 != refs2:
+            report.append(
+                f"references differ: {refs1} vs {refs2}"
+            )
+
+    # 3. Compare sessionIds
+    if "sessionIds" not in excludeFields:
+        ids1 = sorted(summary1.get("sessionIds", []))
+        ids2 = sorted(summary2.get("sessionIds", []))
+        if ids1 != ids2:
+            report.append(
+                f"sessionIds differ: {ids1} vs {ids2}"
+            )
+
+    # 4. Compare sessionSummaries
+    if "sessionSummaries" not in excludeFields:
+        ss1 = summary1.get("sessionSummaries", [])
+        ss2 = summary2.get("sessionSummaries", [])
+
+        if len(ss1) != len(ss2):
+            report.append(
+                f"sessionSummaries count differs: {len(ss1)} vs {len(ss2)}"
+            )
+        else:
+            # Match session summaries by sessionId when available,
+            # otherwise compare by index order.
+            ids1 = summary1.get("sessionIds", [])
+            ids2 = summary2.get("sessionIds", [])
+
+            if len(ids1) == len(ss1) and len(ids2) == len(ss2):
+                # Build lookup by sessionId for summary2
+                lookup2: dict[str, dict] = {}
+                for sid, ss in zip(ids2, ss2):
+                    lookup2[sid] = ss
+
+                for i, sid in enumerate(ids1):
+                    match = lookup2.get(sid)
+                    if match is None:
+                        report.append(
+                            f"sessionSummaries: session {sid} not found in summary2"
+                        )
+                        continue
+                    sub = compareSessionSummary(
+                        ss1[i],
+                        match,
+                        excludeFiles=excludeFiles,
+                        excludeFields=excludeFields,
+                    )
+                    for s in sub:
+                        report.append(f"sessionSummaries[{sid}]: {s}")
+            else:
+                # Fallback: compare by index
+                for i, (s1, s2) in enumerate(zip(ss1, ss2)):
+                    sub = compareSessionSummary(
+                        s1,
+                        s2,
+                        excludeFiles=excludeFiles,
+                        excludeFields=excludeFields,
+                    )
+                    for s in sub:
+                        report.append(f"sessionSummaries[{i}]: {s}")
+
+    return report
diff --git a/src/ndi/util/dataset_summary.py b/src/ndi/util/dataset_summary.py
@@ -0,0 +1,42 @@
+"""ndi_dataset summary utility for symmetry testing.
+
+MATLAB equivalent: ``ndi.util.datasetSummary``
+
+Creates a summary dict of an ``ndi.dataset.Dataset`` object containing key
+fields and properties, intended for symmetry testing between NDI language
+implementations.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .session_summary import sessionSummary
+
+
+def datasetSummary(dataset_obj: Any) -> dict[str, Any]:
+    """Create a summary structure of an ndi.dataset.Dataset object.
+
+    MATLAB equivalent: ``ndi.util.datasetSummary(dataset_obj)``
+
+    Args:
+        dataset_obj: An NDI Dataset object.
+
+    Returns:
+        Dict with keys: numSessions, references, sessionIds,
+        sessionSummaries.
+    """
+    refs, session_ids, *_ = dataset_obj.session_list()
+
+    # Build a session summary for each session in the dataset
+    session_summaries = []
+    for sid in session_ids:
+        sess = dataset_obj.open_session(sid)
+        session_summaries.append(sessionSummary(sess))
+
+    return {
+        "numSessions": len(refs),
+        "references": refs,
+        "sessionIds": session_ids,
+        "sessionSummaries": session_summaries,
+    }
diff --git a/src/ndi/util/ndi_matlab_python_bridge.yaml b/src/ndi/util/ndi_matlab_python_bridge.yaml
@@ -174,6 +174,89 @@ functions:
       Exact match. Replaces NDI sentinel strings for NaN, Infinity,
       and -Infinity in JSON text with Python-compatible representations.
 
+  - name: sessionSummary
+    type: function
+    matlab_path: "+ndi/+util/sessionSummary.m"
+    python_path: "ndi/util/session_summary.py"
+    input_arguments:
+      - name: session_obj
+        type_matlab: "ndi.session"
+        type_python: "Any"
+    output_arguments:
+      - name: summary
+        type_python: "dict[str, Any]"
+    decision_log: >
+      Exact match. Creates a summary dict of an ndi.session object
+      for symmetry testing.
+
+  - name: compareSessionSummary
+    type: function
+    matlab_path: "+ndi/+util/compareSessionSummary.m"
+    python_path: "ndi/util/compare_session_summary.py"
+    input_arguments:
+      - name: summary1
+        type_matlab: "struct"
+        type_python: "dict[str, Any]"
+      - name: summary2
+        type_matlab: "struct"
+        type_python: "dict[str, Any]"
+      - name: excludeFiles
+        type_matlab: "cell array of char"
+        type_python: "list[str] | None"
+        default: "None"
+      - name: excludeFields
+        type_matlab: "cell array of char"
+        type_python: "list[str] | None"
+        default: "None"
+    output_arguments:
+      - name: report
+        type_python: "list[str]"
+    decision_log: >
+      Exact match. Compares two session summaries and returns a list
+      of difference strings. Empty list means summaries match.
+
+  - name: datasetSummary
+    type: function
+    matlab_path: "+ndi/+util/datasetSummary.m"
+    python_path: "ndi/util/dataset_summary.py"
+    input_arguments:
+      - name: dataset_obj
+        type_matlab: "ndi.dataset.Dataset"
+        type_python: "Any"
+    output_arguments:
+      - name: summary
+        type_python: "dict[str, Any]"
+    decision_log: >
+      Exact match. Creates a summary dict of an ndi.dataset.Dataset
+      object for symmetry testing.
+
+  - name: compareDatasetSummary
+    type: function
+    matlab_path: "+ndi/+util/compareDatasetSummary.m"
+    python_path: "ndi/util/compare_dataset_summary.py"
+    input_arguments:
+      - name: summary1
+        type_matlab: "struct"
+        type_python: "dict[str, Any]"
+      - name: summary2
+        type_matlab: "struct"
+        type_python: "dict[str, Any]"
+      - name: excludeFiles
+        type_matlab: "cell array of char"
+        type_python: "list[str] | None"
+        default: "None"
+      - name: excludeFields
+        type_matlab: "cell array of char"
+        type_python: "list[str] | None"
+        default: "None"
+    output_arguments:
+      - name: report
+        type_python: "list[str]"
+    decision_log: >
+      Exact match. Compares two dataset summaries and returns a list
+      of difference strings. Delegates session-level comparison to
+      compareSessionSummary.
+
   - name: unwrapTableCellContent
     type: function
     matlab_path: "+ndi/+util/unwrapTableCellContent.m"
diff --git a/tests/symmetry/make_artifacts/dataset/test_build_dataset.py b/tests/symmetry/make_artifacts/dataset/test_build_dataset.py
@@ -24,7 +24,7 @@
 from ndi.document import Document
 from ndi.query import Query
 from ndi.session.dir import DirSession
-from ndi.util import sessionSummary
+from ndi.util import datasetSummary
 from tests.symmetry.conftest import PYTHON_ARTIFACTS
 
 ARTIFACT_DIR = PYTHON_ARTIFACTS / "dataset" / "buildDataset" / "testBuildDatasetArtifacts"
@@ -46,29 +46,6 @@ def _add_doc_with_file(session: DirSession, doc_number: int) -> None:
     session.database_add(doc)
 
 
-def _dataset_summary(dataset: Dataset) -> dict:
-    """Create a summary structure for a dataset.
-
-    Mirrors MATLAB's ``ndi.symmetry.makeArtifacts.dataset.buildDataset``
-    which writes: numSessions, references, sessionIds, sessionSummaries.
-    """
-    refs, session_ids, *_ = dataset.session_list()
-    num_sessions = len(refs)
-
-    # Build a session summary for each session in the dataset
-    session_summaries = []
-    for sid in session_ids:
-        sess = dataset.open_session(sid)
-        session_summaries.append(sessionSummary(sess))
-
-    return {
-        "numSessions": num_sessions,
-        "references": refs,
-        "sessionIds": session_ids,
-        "sessionSummaries": session_summaries,
-    }
-
-
 class TestBuildDataset:
     """Mirror of ndi.symmetry.makeArtifacts.dataset.buildDataset."""
 
@@ -132,7 +109,7 @@ def test_build_dataset_artifacts(self):
 
         # Write datasetSummary.json – open from artifact_dir so the session
         # path lists files that are actually present (including jsonDocuments).
-        summary = _dataset_summary(artifact_dataset)
+        summary = datasetSummary(artifact_dataset)
         summary_json = json.dumps(summary, indent=2, allow_nan=True)
         summary_path = artifact_dir / "datasetSummary.json"
         summary_path.write_text(summary_json, encoding="utf-8")
diff --git a/tests/symmetry/make_artifacts/dataset/test_download_ingested.py b/tests/symmetry/make_artifacts/dataset/test_download_ingested.py
@@ -24,7 +24,7 @@
 
 from ndi.dataset import Dataset
 from ndi.query import Query
-from ndi.util import sessionSummary
+from ndi.util import datasetSummary
 from tests.symmetry.conftest import PYTHON_ARTIFACTS
 
 ARTIFACT_DIR = PYTHON_ARTIFACTS / "dataset" / "downloadIngested" / "testDownloadIngestedArtifacts"
@@ -99,31 +99,17 @@ def test_download_ingested_artifacts(self):
         # Open the dataset
         dataset = Dataset(dataset_path)
 
-        # Get session list
-        ref_list, id_list, *_ = dataset.session_list()
-        num_sessions = len(ref_list)
+        # Build the dataset summary using the public utility
+        dataset_summary = datasetSummary(dataset)
 
-        # Build session summaries for each session
-        session_summaries = []
-        for sid in id_list:
-            sess = dataset.open_session(sid)
-            session_summaries.append(sessionSummary(sess))
-
-        # Record document counts per session
+        # Record document counts per session (extra field for this test)
+        _ref_list, id_list, *_ = dataset.session_list()
         document_counts = []
         for sid in id_list:
             sess = dataset.open_session(sid)
             docs = sess.database_search(Query("base.id").match("(.*)"))
             document_counts.append({"sessionId": sid, "count": len(docs)})
-
-        # Build the dataset summary
-        dataset_summary = {
-            "numSessions": num_sessions,
-            "references": ref_list,
-            "sessionIds": id_list,
-            "sessionSummaries": session_summaries,
-            "documentCounts": document_counts,
-        }
+        dataset_summary["documentCounts"] = document_counts
 
         # Write datasetSummary.json
         summary_json = json.dumps(dataset_summary, indent=2, allow_nan=True)
diff --git a/tests/symmetry/read_artifacts/dataset/test_build_dataset.py b/tests/symmetry/read_artifacts/dataset/test_build_dataset.py
diff --git a/tests/symmetry/read_artifacts/dataset/test_download_ingested.py b/tests/symmetry/read_artifacts/dataset/test_download_ingested.py
diff --git a/tests/test_dataset_summary.py b/tests/test_dataset_summary.py
