Mirror MATLAB ndi.dataset/ndi.dataset.dir architecture in Python

Major restructuring to match the MATLAB codebase:

**ndi.dataset (base class):**
- reference/id() delegate to session (matching MATLAB)
- build_session_info() discovers sessions from session_in_a_dataset docs
  filtered by base.session_id == dataset.id() (matching MATLAB)
- Static methods: repairDatasetSessionInfo, addSessionInfoToDataset,
  removeSessionInfoFromDataset (matching MATLAB)
- database_search() searches session._database directly + linked sessions
- database_add() routes docs by session_id (matching MATLAB)

**ndi.dataset.dir (DatasetDir subclass):**
- Constructor mirrors MATLAB ndi.dataset.dir with 1/2/3-arg forms
- Hidden documents parameter for downloadDataset (MATLAB 3rd arg)
- Session discovery: dataset_session_info → session_in_a_dataset →
  session docs, then re-creates session with correct ref/id
- _ensure_session_tracking() creates session_in_a_dataset docs for
  untracked sessions found in the database
- Uses dataset path directly for session (.ndi/ at root, not .ndi_dataset/)

**ndi.query module-level functions:**
- Added all(), none(), from_search() at module level so ndi.query.all()
  works like MATLAB (where ndi.query is a class, not a module)

**downloadDataset:**
- Now passes documents to DatasetDir constructor (matching MATLAB pattern)
- Reference discovered from session documents, not cloud API name

https://claude.ai/code/session_01ByofAviQrhGPLQ9BLquGjV

Author: claude

src/ndi/__init__.py

@@ -43,7 +43,7 @@
 from .calculator import Calculator
 from .common import PathConstants, get_logger, getLogger, timestamp
 from .database import Database, open_database
-from .dataset import Dataset
+from .dataset import Dataset, DatasetDir
 from .document import Document
 from .documentservice import DocumentService
 from .element import Element
@@ -117,6 +117,7 @@ def version() -> tuple:
     "ElementTimeseries",
     "Neuron",
     "Dataset",
+    "DatasetDir",
     "App",
     "AppDoc",
     "DocExistsAction",

src/ndi/cloud/orchestration.py

@@ -78,25 +78,24 @@ def downloadDataset(
         for dj in doc_jsons:
             updateFileInfoForRemoteFiles(dj, cloud_dataset_id)
 
-    # Convert to Document objects and add to a local Dataset
-    from ndi.dataset import Dataset
+    # Convert to Document objects and create Dataset with them.
+    # Mirrors MATLAB: ndi.dataset.dir([], datasetFolder, ndiDocuments)
+    from ndi.dataset import DatasetDir
 
-    dataset_reference = ds_info.get("name", cloud_dataset_id)
-    dataset = Dataset(target, reference=dataset_reference)
     documents = jsons2documents(doc_jsons)
-    for doc in documents:
+    dataset = DatasetDir("", target, documents=documents)
+
+    # Create remote link document if not already present
+    from ndi.query import Query
+
+    existing = dataset.database_search(Query("").isa("dataset_remote"))
+    if not existing:
+        remote_doc = createRemoteDatasetDoc(cloud_dataset_id, dataset)
         try:
-            dataset._session._database.add(doc)
+            dataset._session._database.add(remote_doc)
         except Exception:
             pass
 
-    # Create remote link document
-    remote_doc = createRemoteDatasetDoc(cloud_dataset_id, dataset)
-    try:
-        dataset._session._database.add(remote_doc)
-    except Exception:
-        pass
-
     # Store cloud client for on-demand file fetching
     dataset.cloud_client = client
 
@@ -175,20 +174,21 @@ def load_dataset_from_json_dir(
             updateFileInfoForRemoteFiles(dj, cloud_dataset_id)
 
     # Create Dataset
-    from ndi.dataset import Dataset
+    from ndi.dataset import DatasetDir
 
     if target_folder is None:
         target = json_path.parent / f"{json_path.name}_dataset"
     else:
         target = Path(target_folder)
     target.mkdir(parents=True, exist_ok=True)
 
-    dataset = Dataset(target)
+    # Convert JSON dicts to Document objects and create dataset with them
+    from .download import jsons2documents as _j2d
 
-    # Use bulk_add for fast insertion (single transaction, no per-doc
-    # duplicate checks).  This bypasses session_id enforcement since
-    # documents come from multiple remote sessions.
-    added, skipped = dataset._session._database._driver.bulk_add(doc_jsons)
+    all_documents = _j2d(doc_jsons)
+    dataset = DatasetDir("", target, documents=all_documents)
+    added = len(all_documents)
+    skipped = 0
 
     # Wire cloud client for on-demand file fetching
     if client is not None:

src/ndi/dataset/__init__.py

@@ -10,13 +10,19 @@
     ndi.dataset.dir  -> ndi.dataset.dir (constructor for directory-based datasets)
 """
 
-from ._dataset import Dataset
+from ._dataset import Dataset as _DatasetBase  # noqa: F401
+from ._dataset import DatasetDir
+
+# For backward compatibility, ``ndi.dataset.Dataset`` is ``DatasetDir``.
+# The base class is available as ``ndi.dataset._DatasetBase`` if needed.
+Dataset = DatasetDir
 
 # MATLAB compatibility: ``ndi.dataset.dir(path)`` creates a directory-based
 # dataset, mirroring the MATLAB constructor ``ndi.dataset.dir``.
-dir = Dataset
+dir = DatasetDir
 
 __all__ = [
     "Dataset",
+    "DatasetDir",
     "dir",
 ]