Port MATLAB features: preferences, profiles, RHD series navigator, MATLAB BYOL

.github/workflows/ci.yml

@@ -55,6 +55,13 @@ jobs:
         env:
           NDI_CLOUD_USERNAME: ${{ secrets.TEST_USER_2_USERNAME }}
           NDI_CLOUD_PASSWORD: ${{ secrets.TEST_USER_2_PASSWORD }}
+          # The CI test account does not have a registered MATLAB
+          # license, so the destructive BYOL tests (DELETE
+          # /users/me/matlab-license) are allowed to run; they will
+          # clean up after themselves. The license guard in
+          # tests/_matlab_license_guard.py enforces this at module-import
+          # time and refuses to run if this variable is unset.
+          NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE: "false"
         run: |
           # Use sys.monitoring (PEP 669) on Python 3.12+ for faster coverage.
           # CTracer (sys.settrace) is catastrophically slow on 3.12 when

.github/workflows/test-symmetry.yml

@@ -23,10 +23,24 @@ jobs:
         with:
           path: NDI-python
 
-      - name: Check out NDI-matlab
+      - name: Check out NDI-matlab (matching branch, fall back to main)
         uses: actions/checkout@v4
         with:
           repository: VH-Lab/NDI-matlab
+          # Use the same branch name as NDI-python so paired feature
+          # branches stay in sync; fall back to main when the matlab
+          # branch does not exist.
+          ref: ${{ github.head_ref || github.ref_name }}
+          path: NDI-matlab
+        continue-on-error: true
+        id: matlab_checkout_branch
+
+      - name: Check out NDI-matlab (main fallback)
+        if: steps.matlab_checkout_branch.outcome == 'failure'
+        uses: actions/checkout@v4
+        with:
+          repository: VH-Lab/NDI-matlab
+          ref: main
           path: NDI-matlab
 
       # ── Runtime setup ──────────────────────────────────────────────────
@@ -67,6 +81,9 @@ jobs:
             https://github.com/Waltham-Data-Science/file-passing/raw/refs/heads/main/69a8705aa9ab25373cdc6563.tgz
 
       # ── Stage 1: MATLAB makeArtifacts ──────────────────────────────────
+      # Scope by directory: matbox.installRequirements points at the
+      # +ndi/+symmetry folder (which has its own narrow requirements.txt),
+      # and TestSuite.fromFolder discovers tests in that directory tree.
       - name: "Stage 1: MATLAB makeArtifacts"
         uses: matlab-actions/run-command@v2
         with:
@@ -75,12 +92,15 @@ jobs:
             addpath(genpath("src"));
             addpath(genpath("tests"));
             addpath(genpath("tools"));
-            matbox.installRequirements(fullfile(pwd, "tests"));
+
+            symDir = fullfile(pwd, "tests", "+ndi", "+symmetry");
+            matbox.installRequirements(symDir);
 
             import matlab.unittest.TestSuite
             import matlab.unittest.TestRunner
 
-            suite = TestSuite.fromPackage("ndi.symmetry.makeArtifacts", "IncludingSubpackages", true);
+            suite = TestSuite.fromFolder( ...
+                fullfile(symDir, "+makeArtifacts"), "IncludingSubfolders", true);
             fprintf("\n=== MATLAB makeArtifacts: %d test(s) ===\n", numel(suite));
             assert(~isempty(suite), "No MATLAB makeArtifacts tests found.")
 
@@ -111,12 +131,15 @@ jobs:
             addpath(genpath("src"));
             addpath(genpath("tests"));
             addpath(genpath("tools"));
-            matbox.installRequirements(fullfile(pwd, "tests"));
+
+            symDir = fullfile(pwd, "tests", "+ndi", "+symmetry");
+            matbox.installRequirements(symDir);
 
             import matlab.unittest.TestSuite
             import matlab.unittest.TestRunner
 
-            suite = TestSuite.fromPackage("ndi.symmetry.readArtifacts", "IncludingSubpackages", true);
+            suite = TestSuite.fromFolder( ...
+                fullfile(symDir, "+readArtifacts"), "IncludingSubfolders", true);
             fprintf("\n=== MATLAB readArtifacts: %d test(s) ===\n", numel(suite));
             assert(~isempty(suite), "No MATLAB readArtifacts tests found.")
 

src/ndi/class_registry.py

@@ -36,6 +36,10 @@ def _build_registry() -> dict[str, type]:
     from .element import ndi_element
     from .file.navigator import ndi_file_navigator
     from .file.navigator.epochdir import ndi_file_navigator_epochdir
+    from .file.navigator.rhd_series import ndi_file_navigator_rhd_series
+    from .file.navigator.rhd_series_epochdir import (
+        ndi_file_navigator_rhd_series_epochdir,
+    )
     from .probe import ndi_probe
     from .probe.timeseries import ndi_probe_timeseries
     from .probe.timeseries_mfdaq import ndi_probe_timeseries_mfdaq
@@ -46,6 +50,9 @@ def _build_registry() -> dict[str, type]:
     from .setup.daq.reader.mfdaq.stimulus.nielsenvisneuropixelsglx import (
         ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx,
     )
+    from .setup.daq.reader.mfdaq.stimulus.rayolab_intanseries import (
+        ndi_setup_daq_reader_mfdaq_stimulus_rayolab_intanseries,
+    )
     from .setup.daq.reader.mfdaq.stimulus.vhaudreybpod import (
         ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod,
     )
@@ -75,6 +82,7 @@ def _build_registry() -> dict[str, type]:
         ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2,
         ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan,
         ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx,
+        ndi_setup_daq_reader_mfdaq_stimulus_rayolab_intanseries,
         ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod,
     ):
         registry[cls.NDI_DAQREADER_CLASS] = cls
@@ -85,6 +93,10 @@ def _build_registry() -> dict[str, type]:
     # File navigators
     registry[ndi_file_navigator.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator
     registry[ndi_file_navigator_epochdir.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_epochdir
+    registry[ndi_file_navigator_rhd_series.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_rhd_series
+    registry[ndi_file_navigator_rhd_series_epochdir.NDI_FILENAVIGATOR_CLASS] = (
+        ndi_file_navigator_rhd_series_epochdir
+    )
     # Custom lab-specific navigators mapped to epochdir until dedicated classes exist
     registry["ndi.setup.file.navigator.vhlab_np_epochdir"] = ndi_file_navigator_epochdir
 

src/ndi/cloud/__init__.py

@@ -29,10 +29,12 @@
 from .auth import (
     authenticate,
     changePassword,
+    isTokenExpired,
     login,
     logout,
     resendConfirmation,
     resetPassword,
+    testLogin,
     verifyUser,
 )
 from .config import CloudConfig
@@ -54,8 +56,10 @@
     "CloudSyncError",
     "CloudUploadError",
     "authenticate",
+    "isTokenExpired",
     "login",
     "logout",
+    "testLogin",
     "changePassword",
     "resetPassword",
     "verifyUser",
@@ -66,6 +70,7 @@
     "syncDataset",
     "uploadSingleFile",
     "fetch_cloud_file",
+    "profile",
 ]
 
 # Lazy imports for symbols that depend on requests.
@@ -81,6 +86,7 @@
     "syncDataset": ("orchestration", "syncDataset"),
     "uploadSingleFile": ("upload", "uploadSingleFile"),
     "fetch_cloud_file": ("filehandler", "fetch_cloud_file"),
+    "profile": ("profile", None),
 }
 
 
@@ -90,5 +96,7 @@ def __getattr__(name: str):
         import importlib
 
         mod = importlib.import_module(f".{module_name}", __name__)
+        if attr is None:
+            return mod
         return getattr(mod, attr)
     raise AttributeError(f"module 'ndi.cloud' has no attribute {name!r}")

src/ndi/cloud/api/datasets.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import time
 from typing import Annotated, Any
 
 from pydantic import SkipValidation, validate_call
@@ -252,3 +253,126 @@ def listDeletedDatasets(
         "/datasets/deleted",
         params={"page": page, "pageSize": page_size},
     )
+
+
+# ---------------------------------------------------------------------------
+# Wait helpers
+# ---------------------------------------------------------------------------
+
+
+def _wait_for_published_state(
+    dataset_id: str,
+    *,
+    desired: bool,
+    timeout: float,
+    initial_interval: float,
+    max_interval: float,
+    backoff_factor: float,
+    client: CloudClient | None,
+) -> dict[str, Any]:
+    """Shared body for :func:`waitForPublished` / :func:`waitForUnpublished`.
+
+    Polls ``getDataset`` at exponentially growing intervals until the
+    dataset's ``isPublished`` field equals *desired* or the overall
+    timeout elapses.
+    """
+    start = time.monotonic()
+    interval = initial_interval
+    last: Any = None
+    while True:
+        elapsed = time.monotonic() - start
+        try:
+            ds = getDataset(dataset_id, client=client)
+            last = ds
+            is_published = bool(ds.get("isPublished", False)) if hasattr(ds, "get") else False
+            if is_published == desired:
+                return ds
+        except Exception:
+            # Treat transient errors as not-yet-reached; let timeout govern.
+            pass
+        if elapsed + interval > timeout:
+            payload: dict[str, Any]
+            if last is not None and hasattr(last, "data") and isinstance(last.data, dict):
+                payload = dict(last.data)
+            elif isinstance(last, dict):
+                payload = dict(last)
+            else:
+                payload = {}
+            payload["state"] = "timeout"
+            payload["elapsed"] = time.monotonic() - start
+            return payload
+        time.sleep(interval)
+        interval = min(interval * backoff_factor, max_interval)
+
+
+@_auto_client
+@validate_call(config=VALIDATE_CONFIG)
+def waitForPublished(
+    dataset_id: CloudId,
+    *,
+    timeout: float = 180.0,
+    initial_interval: float = 2.0,
+    max_interval: float = 30.0,
+    backoff_factor: float = 2.0,
+    client: _Client = None,
+) -> dict[str, Any]:
+    """Poll a dataset until its ``isPublished`` flag is true.
+
+    Repeatedly calls :func:`getDataset` at exponentially growing
+    intervals until the dataset's ``isPublished`` field becomes ``True``
+    or the overall timeout elapses.  The backend flips ``isPublished``
+    from ``False`` to ``True`` only when publishing has completed, so
+    this is the canonical signal that a publish workflow is finished.
+
+    Args:
+        dataset_id: The cloud dataset ID.
+        timeout: Overall deadline in seconds.  Default 180.
+        initial_interval: First sleep between polls (s).  Default 2.
+        max_interval: Cap on the per-poll sleep (s).  Default 30.
+        backoff_factor: Multiplier applied after each poll.  Default 2.
+
+    Returns:
+        The last dataset payload from the server.  On timeout, the
+        returned dict has ``state='timeout'`` and ``elapsed`` set to
+        the wall-clock seconds spent polling.
+
+    MATLAB equivalent: +cloud/+api/+datasets/waitForPublished.m
+    """
+    return _wait_for_published_state(
+        dataset_id,
+        desired=True,
+        timeout=timeout,
+        initial_interval=initial_interval,
+        max_interval=max_interval,
+        backoff_factor=backoff_factor,
+        client=client,
+    )
+
+
+@_auto_client
+@validate_call(config=VALIDATE_CONFIG)
+def waitForUnpublished(
+    dataset_id: CloudId,
+    *,
+    timeout: float = 180.0,
+    initial_interval: float = 2.0,
+    max_interval: float = 30.0,
+    backoff_factor: float = 2.0,
+    client: _Client = None,
+) -> dict[str, Any]:
+    """Poll a dataset until its ``isPublished`` flag is false.
+
+    See :func:`waitForPublished` for the polling semantics; this
+    function waits for the opposite transition.
+
+    MATLAB equivalent: +cloud/+api/+datasets/waitForUnpublished.m
+    """
+    return _wait_for_published_state(
+        dataset_id,
+        desired=False,
+        timeout=timeout,
+        initial_interval=initial_interval,
+        max_interval=max_interval,
+        backoff_factor=backoff_factor,
+        client=client,
+    )