diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b9973..5f9b82e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,13 @@ jobs: env: NDI_CLOUD_USERNAME: ${{ secrets.TEST_USER_2_USERNAME }} NDI_CLOUD_PASSWORD: ${{ secrets.TEST_USER_2_PASSWORD }} + # The CI test account does not have a registered MATLAB + # license, so the destructive BYOL tests (DELETE + # /users/me/matlab-license) are allowed to run; they will + # clean up after themselves. The license guard in + # tests/_matlab_license_guard.py enforces this at module-import + # time and refuses to run if this variable is unset. + NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE: "false" run: | # Use sys.monitoring (PEP 669) on Python 3.12+ for faster coverage. # CTracer (sys.settrace) is catastrophically slow on 3.12 when diff --git a/.github/workflows/test-symmetry.yml b/.github/workflows/test-symmetry.yml index 38487cd..6b5dcad 100644 --- a/.github/workflows/test-symmetry.yml +++ b/.github/workflows/test-symmetry.yml @@ -23,10 +23,24 @@ jobs: with: path: NDI-python - - name: Check out NDI-matlab + - name: Check out NDI-matlab (matching branch, fall back to main) uses: actions/checkout@v4 with: repository: VH-Lab/NDI-matlab + # Use the same branch name as NDI-python so paired feature + # branches stay in sync; fall back to main when the matlab + # branch does not exist. + ref: ${{ github.head_ref || github.ref_name }} + path: NDI-matlab + continue-on-error: true + id: matlab_checkout_branch + + - name: Check out NDI-matlab (main fallback) + if: steps.matlab_checkout_branch.outcome == 'failure' + uses: actions/checkout@v4 + with: + repository: VH-Lab/NDI-matlab + ref: main path: NDI-matlab # ── Runtime setup ────────────────────────────────────────────────── @@ -67,6 +81,9 @@ jobs: https://github.com/Waltham-Data-Science/file-passing/raw/refs/heads/main/69a8705aa9ab25373cdc6563.tgz # ── Stage 1: MATLAB makeArtifacts ────────────────────────────────── + # Scope by directory: matbox.installRequirements points at the + # +ndi/+symmetry folder (which has its own narrow requirements.txt), + # and TestSuite.fromFolder discovers tests in that directory tree. - name: "Stage 1: MATLAB makeArtifacts" uses: matlab-actions/run-command@v2 with: @@ -75,12 +92,15 @@ jobs: addpath(genpath("src")); addpath(genpath("tests")); addpath(genpath("tools")); - matbox.installRequirements(fullfile(pwd, "tests")); + + symDir = fullfile(pwd, "tests", "+ndi", "+symmetry"); + matbox.installRequirements(symDir); import matlab.unittest.TestSuite import matlab.unittest.TestRunner - suite = TestSuite.fromPackage("ndi.symmetry.makeArtifacts", "IncludingSubpackages", true); + suite = TestSuite.fromFolder( ... + fullfile(symDir, "+makeArtifacts"), "IncludingSubfolders", true); fprintf("\n=== MATLAB makeArtifacts: %d test(s) ===\n", numel(suite)); assert(~isempty(suite), "No MATLAB makeArtifacts tests found.") @@ -111,12 +131,15 @@ jobs: addpath(genpath("src")); addpath(genpath("tests")); addpath(genpath("tools")); - matbox.installRequirements(fullfile(pwd, "tests")); + + symDir = fullfile(pwd, "tests", "+ndi", "+symmetry"); + matbox.installRequirements(symDir); import matlab.unittest.TestSuite import matlab.unittest.TestRunner - suite = TestSuite.fromPackage("ndi.symmetry.readArtifacts", "IncludingSubpackages", true); + suite = TestSuite.fromFolder( ... + fullfile(symDir, "+readArtifacts"), "IncludingSubfolders", true); fprintf("\n=== MATLAB readArtifacts: %d test(s) ===\n", numel(suite)); assert(~isempty(suite), "No MATLAB readArtifacts tests found.") diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index 964f995..038a30e 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -36,6 +36,10 @@ def _build_registry() -> dict[str, type]: from .element import ndi_element from .file.navigator import ndi_file_navigator from .file.navigator.epochdir import ndi_file_navigator_epochdir + from .file.navigator.rhd_series import ndi_file_navigator_rhd_series + from .file.navigator.rhd_series_epochdir import ( + ndi_file_navigator_rhd_series_epochdir, + ) from .probe import ndi_probe from .probe.timeseries import ndi_probe_timeseries from .probe.timeseries_mfdaq import ndi_probe_timeseries_mfdaq @@ -46,6 +50,9 @@ def _build_registry() -> dict[str, type]: from .setup.daq.reader.mfdaq.stimulus.nielsenvisneuropixelsglx import ( ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx, ) + from .setup.daq.reader.mfdaq.stimulus.rayolab_intanseries import ( + ndi_setup_daq_reader_mfdaq_stimulus_rayolab_intanseries, + ) from .setup.daq.reader.mfdaq.stimulus.vhaudreybpod import ( ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, ) @@ -75,6 +82,7 @@ def _build_registry() -> dict[str, type]: ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx, + ndi_setup_daq_reader_mfdaq_stimulus_rayolab_intanseries, ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, ): registry[cls.NDI_DAQREADER_CLASS] = cls @@ -85,6 +93,10 @@ def _build_registry() -> dict[str, type]: # File navigators registry[ndi_file_navigator.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator registry[ndi_file_navigator_epochdir.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_epochdir + registry[ndi_file_navigator_rhd_series.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_rhd_series + registry[ndi_file_navigator_rhd_series_epochdir.NDI_FILENAVIGATOR_CLASS] = ( + ndi_file_navigator_rhd_series_epochdir + ) # Custom lab-specific navigators mapped to epochdir until dedicated classes exist registry["ndi.setup.file.navigator.vhlab_np_epochdir"] = ndi_file_navigator_epochdir diff --git a/src/ndi/cloud/__init__.py b/src/ndi/cloud/__init__.py index 217b426..963a502 100644 --- a/src/ndi/cloud/__init__.py +++ b/src/ndi/cloud/__init__.py @@ -29,10 +29,12 @@ from .auth import ( authenticate, changePassword, + isTokenExpired, login, logout, resendConfirmation, resetPassword, + testLogin, verifyUser, ) from .config import CloudConfig @@ -54,8 +56,10 @@ "CloudSyncError", "CloudUploadError", "authenticate", + "isTokenExpired", "login", "logout", + "testLogin", "changePassword", "resetPassword", "verifyUser", @@ -66,6 +70,7 @@ "syncDataset", "uploadSingleFile", "fetch_cloud_file", + "profile", ] # Lazy imports for symbols that depend on requests. @@ -81,6 +86,7 @@ "syncDataset": ("orchestration", "syncDataset"), "uploadSingleFile": ("upload", "uploadSingleFile"), "fetch_cloud_file": ("filehandler", "fetch_cloud_file"), + "profile": ("profile", None), } @@ -90,5 +96,7 @@ def __getattr__(name: str): import importlib mod = importlib.import_module(f".{module_name}", __name__) + if attr is None: + return mod return getattr(mod, attr) raise AttributeError(f"module 'ndi.cloud' has no attribute {name!r}") diff --git a/src/ndi/cloud/api/datasets.py b/src/ndi/cloud/api/datasets.py index aa233c9..7c45c65 100644 --- a/src/ndi/cloud/api/datasets.py +++ b/src/ndi/cloud/api/datasets.py @@ -11,6 +11,7 @@ from __future__ import annotations +import time from typing import Annotated, Any from pydantic import SkipValidation, validate_call @@ -252,3 +253,126 @@ def listDeletedDatasets( "/datasets/deleted", params={"page": page, "pageSize": page_size}, ) + + +# --------------------------------------------------------------------------- +# Wait helpers +# --------------------------------------------------------------------------- + + +def _wait_for_published_state( + dataset_id: str, + *, + desired: bool, + timeout: float, + initial_interval: float, + max_interval: float, + backoff_factor: float, + client: CloudClient | None, +) -> dict[str, Any]: + """Shared body for :func:`waitForPublished` / :func:`waitForUnpublished`. + + Polls ``getDataset`` at exponentially growing intervals until the + dataset's ``isPublished`` field equals *desired* or the overall + timeout elapses. + """ + start = time.monotonic() + interval = initial_interval + last: Any = None + while True: + elapsed = time.monotonic() - start + try: + ds = getDataset(dataset_id, client=client) + last = ds + is_published = bool(ds.get("isPublished", False)) if hasattr(ds, "get") else False + if is_published == desired: + return ds + except Exception: + # Treat transient errors as not-yet-reached; let timeout govern. + pass + if elapsed + interval > timeout: + payload: dict[str, Any] + if last is not None and hasattr(last, "data") and isinstance(last.data, dict): + payload = dict(last.data) + elif isinstance(last, dict): + payload = dict(last) + else: + payload = {} + payload["state"] = "timeout" + payload["elapsed"] = time.monotonic() - start + return payload + time.sleep(interval) + interval = min(interval * backoff_factor, max_interval) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def waitForPublished( + dataset_id: CloudId, + *, + timeout: float = 180.0, + initial_interval: float = 2.0, + max_interval: float = 30.0, + backoff_factor: float = 2.0, + client: _Client = None, +) -> dict[str, Any]: + """Poll a dataset until its ``isPublished`` flag is true. + + Repeatedly calls :func:`getDataset` at exponentially growing + intervals until the dataset's ``isPublished`` field becomes ``True`` + or the overall timeout elapses. The backend flips ``isPublished`` + from ``False`` to ``True`` only when publishing has completed, so + this is the canonical signal that a publish workflow is finished. + + Args: + dataset_id: The cloud dataset ID. + timeout: Overall deadline in seconds. Default 180. + initial_interval: First sleep between polls (s). Default 2. + max_interval: Cap on the per-poll sleep (s). Default 30. + backoff_factor: Multiplier applied after each poll. Default 2. + + Returns: + The last dataset payload from the server. On timeout, the + returned dict has ``state='timeout'`` and ``elapsed`` set to + the wall-clock seconds spent polling. + + MATLAB equivalent: +cloud/+api/+datasets/waitForPublished.m + """ + return _wait_for_published_state( + dataset_id, + desired=True, + timeout=timeout, + initial_interval=initial_interval, + max_interval=max_interval, + backoff_factor=backoff_factor, + client=client, + ) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def waitForUnpublished( + dataset_id: CloudId, + *, + timeout: float = 180.0, + initial_interval: float = 2.0, + max_interval: float = 30.0, + backoff_factor: float = 2.0, + client: _Client = None, +) -> dict[str, Any]: + """Poll a dataset until its ``isPublished`` flag is false. + + See :func:`waitForPublished` for the polling semantics; this + function waits for the opposite transition. + + MATLAB equivalent: +cloud/+api/+datasets/waitForUnpublished.m + """ + return _wait_for_published_state( + dataset_id, + desired=False, + timeout=timeout, + initial_interval=initial_interval, + max_interval=max_interval, + backoff_factor=backoff_factor, + client=client, + ) diff --git a/src/ndi/cloud/api/files.py b/src/ndi/cloud/api/files.py index 83311c7..494b322 100644 --- a/src/ndi/cloud/api/files.py +++ b/src/ndi/cloud/api/files.py @@ -10,8 +10,9 @@ from __future__ import annotations +import time from pathlib import Path -from typing import Annotated, Any +from typing import Annotated, Any, Literal from pydantic import SkipValidation, validate_call @@ -20,6 +21,10 @@ _Client = Annotated[CloudClient | None, SkipValidation()] +# Terminal job states reported by the bulk-upload service. +_TERMINAL_BULK_STATES = ("complete", "failed") +_ACTIVE_BULK_STATES = ("queued", "extracting") + @_auto_client @validate_call(config=VALIDATE_CONFIG) @@ -68,19 +73,37 @@ def putFiles( url: NonEmptyStr, file_path: FilePath, timeout: int = 120, + *, + job_id: str = "", + wait_for_completion: bool = False, + completion_timeout: float = 60.0, ) -> bool: """PUT a local file to a presigned S3 URL. Args: url: Presigned URL. file_path: Path to file on disk. - timeout: Request timeout in seconds. + timeout: Per-request timeout in seconds for the PUT. + job_id: Bulk-upload job identifier returned by + :func:`getFileCollectionUploadURL`. Only meaningful for + bulk (zip) uploads; ignored for single-file uploads. + wait_for_completion: If True, after a successful PUT the + function polls :func:`waitForBulkUpload` and only returns + once the server has finished extracting the zip (or the + timeout is hit). Requires a non-empty ``job_id``. + Single-file uploads have no server-side job to wait on; + the signed PUT returning 200 already means done. + completion_timeout: Overall wait-for-completion deadline, in + seconds. Default 60. Returns: - True on success. + True on success (and, when ``wait_for_completion`` is True, the + server-side bulk extraction job reached state ``'complete'``). Raises: CloudUploadError: On failure. + + MATLAB equivalent: +cloud/+api/+files/putFiles.m """ import requests @@ -95,9 +118,18 @@ def putFiles( timeout=timeout, ) - if resp.status_code == 200: - return True - raise CloudUploadError(f"File upload failed (HTTP {resp.status_code}): {resp.text}") + if resp.status_code != 200: + raise CloudUploadError(f"File upload failed (HTTP {resp.status_code}): {resp.text}") + + if wait_for_completion: + if not job_id: + # Single-file upload: nothing server-side to wait on. + return True + final = waitForBulkUpload(job_id, timeout=completion_timeout) + state = final.get("state", "") if hasattr(final, "get") else "" + return state == "complete" + + return True @validate_call @@ -222,8 +254,16 @@ def getFileCollectionUploadURL( dataset_id: CloudId, *, client: _Client = None, -) -> str: - """Get a presigned URL for bulk file collection upload. +) -> dict[str, Any]: + """Get a presigned URL (and ``jobId``) for bulk file collection upload. + + Returns a dict with keys ``url`` (the pre-signed PUT URL for the zip + archive) and ``jobId`` (identifier of the server-side extraction + job). Pass ``jobId`` to :func:`waitForBulkUpload` -- or to + :func:`putFiles` with ``wait_for_completion=True`` -- to wait for + the server to finish extracting the zip before attempting to + download the extracted files. ``jobId`` is an empty string for + older server versions that don't return one. MATLAB equivalent: +cloud/+api/+files/getFileCollectionUploadURL.m """ @@ -232,4 +272,202 @@ def getFileCollectionUploadURL( organizationId=org_id, datasetId=dataset_id, ) - return result.get("url", "") + url = result.get("url", "") if hasattr(result, "get") else "" + job_id = result.get("jobId", "") if hasattr(result, "get") else "" + return {"url": url, "jobId": job_id} + + +# --------------------------------------------------------------------------- +# Bulk-upload status / wait helpers +# --------------------------------------------------------------------------- + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def getBulkUploadStatus( + job_id: NonEmptyStr, + *, + client: _Client = None, +) -> dict[str, Any]: + """GET /bulk-uploads/{jobId} -- Get the state of a bulk file-upload job. + + Returns a dict with fields ``jobId``, ``datasetId``, ``state``, + ``createdAt``, ``startedAt``, ``completedAt``, ``filesExtracted``, + ``totalFiles``, ``error``. + + MATLAB equivalent: +cloud/+api/+files/getBulkUploadStatus.m + """ + return client.get("/bulk-uploads/{jobId}", jobId=job_id) + + +# Convenience alias matching the MATLAB doc-string label. +bulkUploadsJobInfo = getBulkUploadStatus + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def listActiveBulkUploads( + dataset_id: CloudId, + *, + state: Literal["active", "all", "queued", "extracting", "complete", "failed"] = "active", + client: _Client = None, +) -> dict[str, Any]: + """GET /datasets/{datasetId}/bulk-uploads[?state=...] + + List bulk upload jobs the server is tracking for *dataset_id*. + + Args: + dataset_id: The cloud dataset ID. + state: Filter by job state. One of ``'active'`` (default; + ``queued + extracting``), ``'all'`` (includes recent + history), ``'queued'``, ``'extracting'``, ``'complete'``, + ``'failed'``. + + Returns: + Dict with fields ``datasetId`` and ``jobs`` (a list of job + status dicts; see :func:`getBulkUploadStatus` for fields). + + MATLAB equivalent: +cloud/+api/+files/listActiveBulkUploads.m + """ + return client.get( + "/datasets/{datasetId}/bulk-uploads", + params={"state": state}, + datasetId=dataset_id, + ) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def waitForBulkUpload( + job_id: NonEmptyStr, + *, + timeout: float = 60.0, + initial_interval: float = 1.0, + max_interval: float = 30.0, + backoff_factor: float = 2.0, + client: _Client = None, +) -> dict[str, Any]: + """Poll a bulk file-upload job until it finishes or times out. + + Repeatedly calls :func:`getBulkUploadStatus` at exponentially + growing intervals until the job reaches a terminal state + (``'complete'`` or ``'failed'``) or the overall timeout elapses. + + Returns: + The last status dict from the server. On timeout, the + returned dict has ``state='timeout'`` and ``elapsed`` set to + the wall-clock seconds spent polling. + + MATLAB equivalent: +cloud/+api/+files/waitForBulkUpload.m + """ + start = time.monotonic() + interval = initial_interval + last: Any = None + while True: + elapsed = time.monotonic() - start + try: + status = getBulkUploadStatus(job_id, client=client) + last = status + state = status.get("state", "") if hasattr(status, "get") else "" + if state in _TERMINAL_BULK_STATES: + return status + except Exception: + pass + if elapsed + interval > timeout: + payload: dict[str, Any] + if last is not None and hasattr(last, "data") and isinstance(last.data, dict): + payload = dict(last.data) + elif isinstance(last, dict): + payload = dict(last) + else: + payload = {} + payload["state"] = "timeout" + payload["elapsed"] = time.monotonic() - start + return payload + time.sleep(interval) + interval = min(interval * backoff_factor, max_interval) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def waitForAllBulkUploads( + dataset_id: CloudId, + *, + timeout: float = 300.0, + initial_interval: float = 1.0, + max_interval: float = 30.0, + backoff_factor: float = 2.0, + require_all_complete: bool = True, + client: _Client = None, +) -> dict[str, Any]: + """Wait for every bulk-upload job on a dataset to finish. + + Polls :func:`listActiveBulkUploads` at exponentially growing + intervals until no active (queued + extracting) bulk-upload jobs + remain on the dataset or the overall timeout elapses. Intended for + use at sync-pipeline boundaries: before inventorying remote state, + callers should wait for any in-flight extractions so the inventory + is stable. + + Args: + dataset_id: The cloud dataset ID. + timeout: Overall deadline in seconds. Default 300. + initial_interval: First sleep between polls (s). Default 1. + max_interval: Cap on the per-poll sleep (s). Default 30. + backoff_factor: Multiplier applied after each poll. Default 2. + require_all_complete: If True, return ``state='failed'`` when + any job on the dataset ended in ``'failed'``. If False, + return ``state='complete'`` as soon as the active set + drains, regardless of failure history. Default True. + + Returns: + Dict describing the final state with fields ``state`` + (``'complete'``, ``'failed'``, or ``'timeout'``), ``jobs`` + (active or failed jobs), and ``elapsed`` (wall-clock seconds). + + MATLAB equivalent: +cloud/+api/+files/waitForAllBulkUploads.m + """ + start = time.monotonic() + interval = initial_interval + last_jobs: list[dict[str, Any]] = [] + while True: + elapsed = time.monotonic() - start + try: + scope = "all" if require_all_complete else "active" + listing = listActiveBulkUploads(dataset_id, state=scope, client=client) + jobs = listing.get("jobs", []) if hasattr(listing, "get") else [] + last_jobs = list(jobs) if jobs else [] + + active_jobs = [ + j + for j in last_jobs + if (j.get("state", "") if isinstance(j, dict) else "") in _ACTIVE_BULK_STATES + ] + failed_jobs = [ + j + for j in last_jobs + if (j.get("state", "") if isinstance(j, dict) else "") == "failed" + ] + + if not active_jobs: + if require_all_complete and failed_jobs: + return { + "state": "failed", + "jobs": failed_jobs, + "elapsed": time.monotonic() - start, + } + return { + "state": "complete", + "jobs": [], + "elapsed": time.monotonic() - start, + } + except Exception: + active_jobs = last_jobs # treat error as still active; let timeout govern + if elapsed + interval > timeout: + return { + "state": "timeout", + "jobs": last_jobs, + "elapsed": time.monotonic() - start, + } + time.sleep(interval) + interval = min(interval * backoff_factor, max_interval) diff --git a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml index edbe64d..750b5b0 100644 --- a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml @@ -29,6 +29,13 @@ infrastructure: python_path: "ndi/cloud/api/documents.py" decision_log: "Python-only helper to normalize search structures." + - name: _wait_for_published_state + type: internal_helper + python_path: "ndi/cloud/api/datasets.py" + decision_log: > + Python-only shared body for waitForPublished / waitForUnpublished. + MATLAB has separate implementation classes for each. + # ============================================================================= # Functions — datasets.py # Maps to: +cloud/+api/+datasets/ @@ -310,6 +317,78 @@ functions: type_python: "dict[str, Any]" decision_log: "Exact match." + - name: waitForPublished + matlab_path: "+ndi/+cloud/+api/+datasets/waitForPublished.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/datasets.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: timeout + type_matlab: "double" + type_python: "float" + default: "180.0" + - name: initial_interval + type_matlab: "double" + type_python: "float" + default: "2.0" + - name: max_interval + type_matlab: "double" + type_python: "float" + default: "30.0" + - name: backoff_factor + type_matlab: "double" + type_python: "float" + default: "2.0" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + Polls getDataset with exponential backoff until isPublished=True or + the timeout elapses. On timeout the returned dict has + state='timeout' and elapsed=, matching MATLAB. Python + returns only the dataset payload; MATLAB returns + (b, answer, apiResponse, apiURL). + + - name: waitForUnpublished + matlab_path: "+ndi/+cloud/+api/+datasets/waitForUnpublished.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/datasets.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: timeout + type_matlab: "double" + type_python: "float" + default: "180.0" + - name: initial_interval + type_matlab: "double" + type_python: "float" + default: "2.0" + - name: max_interval + type_matlab: "double" + type_python: "float" + default: "30.0" + - name: backoff_factor + type_matlab: "double" + type_python: "float" + default: "2.0" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + Mirror of waitForPublished but waits for isPublished=False. + # --- documents.py --- - name: getDocument matlab_path: "+ndi/+cloud/+api/+documents/getDocument.m" @@ -708,7 +787,7 @@ functions: - name: putFiles matlab_path: "+ndi/+cloud/+api/+files/putFiles.m" - matlab_last_sync_hash: "eefb3a4a" + matlab_last_sync_hash: "2566fe4d" python_path: "ndi/cloud/api/files.py" input_arguments: - name: url @@ -720,10 +799,30 @@ functions: - name: timeout type_python: "int" default: "120" + - name: job_id + type_matlab: "string" + type_python: "str" + default: "''" + - name: wait_for_completion + type_matlab: "logical" + type_python: "bool" + default: "False" + - name: completion_timeout + type_matlab: "double" + type_python: "float" + default: "60.0" output_arguments: - name: success type_python: "bool" - decision_log: "Exact match." + decision_log: > + Synchronized with MATLAB main as of 2026-05-11 (matlab HEAD + 2566fe4d). Added job_id and wait_for_completion options to mirror + the new MATLAB bulk-upload "fire-and-wait" pattern: after a + successful PUT, when wait_for_completion=True and job_id is set, + the function polls waitForBulkUpload until the server-side + extraction job reaches state 'complete'. The MATLAB + 'useCurl' option is omitted from the Python port; requests does + the right thing on all supported platforms. - name: putFileBytes matlab_path: "N/A" @@ -797,7 +896,7 @@ functions: - name: getFileCollectionUploadURL matlab_path: "+ndi/+cloud/+api/+files/getFileCollectionUploadURL.m" - matlab_last_sync_hash: "9b75c0fe" + matlab_last_sync_hash: "2566fe4d" python_path: "ndi/cloud/api/files.py" input_arguments: - name: org_id @@ -808,9 +907,142 @@ functions: type_python: "_Client" default: "None" output_arguments: - - name: url - type_python: "str" - decision_log: "Exact match." + - name: info + type_python: "dict[str, Any]" + decision_log: > + Updated to return a dict {url, jobId} instead of just the url so + that callers can pass jobId to waitForBulkUpload / putFiles + (matches MATLAB main as of 2026-05-11). Older callers that only + need the url should use result['url']. + + - name: getBulkUploadStatus + matlab_path: "+ndi/+cloud/+api/+files/getBulkUploadStatus.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: job_id + type_matlab: "string" + type_python: "NonEmptyStr" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + GET /bulk-uploads/{jobId}. Returns the server-side extraction job + state. Python module also exposes the alias bulkUploadsJobInfo + (matches MATLAB's secondary helper name). + + - name: bulkUploadsJobInfo + matlab_path: "+ndi/+cloud/+api/+files/getBulkUploadStatus.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/files.py" + python_qualified: "ndi.cloud.api.files.bulkUploadsJobInfo" + input_arguments: + - name: job_id + type_python: "NonEmptyStr" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Alias for getBulkUploadStatus, matching the MATLAB helper name + introduced in commit a6056d1500. + + - name: listActiveBulkUploads + matlab_path: "+ndi/+cloud/+api/+files/listActiveBulkUploads.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: state + type_matlab: "string" + type_python: "Literal['active','all','queued','extracting','complete','failed']" + default: "'active'" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + GET /datasets/{datasetId}/bulk-uploads[?state=...]. + + - name: waitForBulkUpload + matlab_path: "+ndi/+cloud/+api/+files/waitForBulkUpload.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: job_id + type_matlab: "string" + type_python: "NonEmptyStr" + - name: timeout + type_python: "float" + default: "60.0" + - name: initial_interval + type_python: "float" + default: "1.0" + - name: max_interval + type_python: "float" + default: "30.0" + - name: backoff_factor + type_python: "float" + default: "2.0" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + Polls getBulkUploadStatus with exponential backoff until the job + reaches a terminal state (complete | failed) or the timeout + elapses. On timeout the returned dict has state='timeout' and + elapsed=. + + - name: waitForAllBulkUploads + matlab_path: "+ndi/+cloud/+api/+files/waitForAllBulkUploads.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/files.py" + input_arguments: + - name: dataset_id + type_matlab: "string" + type_python: "CloudId" + - name: timeout + type_python: "float" + default: "300.0" + - name: initial_interval + type_python: "float" + default: "1.0" + - name: max_interval + type_python: "float" + default: "30.0" + - name: backoff_factor + type_python: "float" + default: "2.0" + - name: require_all_complete + type_matlab: "logical" + type_python: "bool" + default: "True" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: result + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + Polls listActiveBulkUploads until the active set drains. + Sync-pipeline callers use this to ensure remote inventories are + stable before reading them. # --- users.py --- - name: createUser @@ -866,6 +1098,113 @@ functions: MATLAB uses PascalCase 'GetUser'. Python preserves this exactly per strict naming policy. + - name: getMatlabLicense + matlab_path: "+ndi/+cloud/+api/+users/getMatlabLicense.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/users.py" + input_arguments: + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + GET /users/me/matlab-license. Part of the MATLAB BYOL license + API. Python keeps the MATLAB camelCase name verbatim + (per strict naming policy) even though it refers to MATLAB + specifically. + Test port (2026-05-11): MATLAB MatlabLicenseTest.testGetMatlabLicense + ported to tests/test_cloud_matlab_license.py::TestMatlabLicense:: + test_getMatlabLicense. Read-only branch runs under both modes of + the NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE guard. + + - name: allocateMatlabLicenseMac + matlab_path: "+ndi/+cloud/+api/+users/allocateMatlabLicenseMac.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/users.py" + input_arguments: + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + POST /users/me/matlab-license. Idempotent allocation of an AWS + ENI/MAC for dedicated MATLAB BYOL. + Test port (2026-05-11): exercised by + tests/test_cloud_matlab_license.py::TestMatlabLicense:: + test_allocate_and_clear_lifecycle (skipped when + NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE=true to preserve any + existing registration). + + - name: setMatlabLicense + matlab_path: "+ndi/+cloud/+api/+users/setMatlabLicense.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/users.py" + input_arguments: + - name: license_file + type_matlab: "string" + type_python: "NonEmptyStr" + - name: mode + type_matlab: "string" + type_python: "Literal['dedicated','network']" + default: "'dedicated'" + - name: release + type_matlab: "string" + type_python: "str" + default: "''" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + PUT /users/me/matlab-license. Accepts either the contents of the + .lic file as a string, or a path to a .lic file on disk + (auto-detected: a single-line argument that exists as a file is + read in). + Test port (2026-05-11): negative-path covered by + tests/test_cloud_matlab_license.py::TestMatlabLicense:: + test_setMatlabLicense_rejects_invalid_file (HTTP 400 on bogus + lic; skipped under HAS_LICENSE=true). + + - name: clearMatlabLicense + matlab_path: "+ndi/+cloud/+api/+users/clearMatlabLicense.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/api/users.py" + input_arguments: + - name: release + type_matlab: "string" + type_python: "str" + default: "''" + - name: client + type_python: "_Client" + default: "None" + output_arguments: + - name: status + type_python: "dict[str, Any]" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + DELETE /users/me/matlab-license. Without release, fully clears + the user's registration; with release, only that release entry + is removed from a dedicated registration. + Test port (2026-05-11): exercised by + tests/test_cloud_matlab_license.py::TestMatlabLicense:: + test_allocate_and_clear_lifecycle plus the per-test + destructive_teardown finalizer (mirror of MATLAB + TestMethodTeardown.maybeClear). The teardown never runs when + NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE=true; see + tests/_matlab_license_guard.py for the rationale (refusing to + proceed without an explicit env-var declaration prevents a + misconfigured CI from silently deleting a real registered + license). + # --- compute.py --- - name: startSession matlab_path: "+ndi/+cloud/+api/+compute/startSession.m" diff --git a/src/ndi/cloud/api/users.py b/src/ndi/cloud/api/users.py index 570449e..e9a33ff 100644 --- a/src/ndi/cloud/api/users.py +++ b/src/ndi/cloud/api/users.py @@ -10,7 +10,8 @@ from __future__ import annotations -from typing import Annotated, Any +from pathlib import Path +from typing import Annotated, Any, Literal from pydantic import SkipValidation, validate_call @@ -50,3 +51,124 @@ def me(*, client: _Client = None) -> dict[str, Any]: def GetUser(user_id: NonEmptyStr, *, client: _Client = None) -> dict[str, Any]: """GET /users/{userId}""" return client.get("/users/{userId}", userId=user_id) + + +# --------------------------------------------------------------------------- +# MATLAB BYOL license wrappers +# +# MATLAB equivalents: +# +ndi/+cloud/+api/+users/getMatlabLicense.m +# +ndi/+cloud/+api/+users/setMatlabLicense.m +# +ndi/+cloud/+api/+users/clearMatlabLicense.m +# +ndi/+cloud/+api/+users/allocateMatlabLicenseMac.m +# --------------------------------------------------------------------------- + + +def _unwrap(result: Any) -> dict[str, Any]: + # CloudClient verbs return APIResponse, whose .data is the parsed JSON + # body. The BYOL wrappers advertise dict[str, Any] in their signature + # and tests assert isinstance(result, dict), so unwrap once here. + if isinstance(result, dict): + return result + data = getattr(result, "data", None) + if isinstance(data, dict): + return data + return {} + + +@_auto_client +def getMatlabLicense(*, client: _Client = None) -> dict[str, Any]: + """GET /users/me/matlab-license -- Retrieve the current MATLAB BYOL status. + + Returns the ``MatlabLicenseStatus`` document (``mode``, ``eniId``, + ``macAddress``, ``subnetId``, ``registeredAt``, ``files``, + ``instructions``). When no license is registered the server still + returns 200 with ``mode == ""`` / ``None`` and an empty ``files`` + array. + + MATLAB equivalent: +cloud/+api/+users/getMatlabLicense.m + """ + return _unwrap(client.get("/users/me/matlab-license")) + + +@_auto_client +def allocateMatlabLicenseMac(*, client: _Client = None) -> dict[str, Any]: + """POST /users/me/matlab-license -- Allocate an AWS ENI/MAC for dedicated MATLAB BYOL. + + Idempotent: returns the existing MAC if a dedicated registration + already exists; otherwise allocates a new ENI in the configured + subnet and returns its MAC address. + + The caller registers the returned MAC with MathWorks to obtain a + ``.lic`` file, then uploads it via :func:`setMatlabLicense` with the + matching ``release`` tag. + + Conflicts: returns HTTP 409 if a network license is currently + registered; clear it first via :func:`clearMatlabLicense`. + + MATLAB equivalent: +cloud/+api/+users/allocateMatlabLicenseMac.m + """ + return _unwrap(client.post("/users/me/matlab-license")) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def setMatlabLicense( + license_file: NonEmptyStr, + *, + mode: Literal["dedicated", "network"] = "dedicated", + release: str = "", + client: _Client = None, +) -> dict[str, Any]: + """PUT /users/me/matlab-license -- Upload a MATLAB BYOL license file. + + Args: + license_file: Either the contents of the ``.lic`` file as a + string, or a path to a ``.lic`` file on disk (auto-detected: + a single-line argument that exists as a file is read in). + mode: ``"dedicated"`` (default) — per-MAC license; requires a + ``release`` tag (e.g. ``"R2024b"``) and a prior call to + :func:`allocateMatlabLicenseMac` whose MAC the lic file's + HOSTID matches. ``"network"`` — license-server file + containing a SERVER line; must not supply ``release``. + release: Release tag (e.g. ``"R2024b"``) for dedicated mode. + + MATLAB equivalent: +cloud/+api/+users/setMatlabLicense.m + """ + # If license_file looks like a path that exists on disk, read it in. + license_text = license_file + if "\n" not in license_file: + try: + p = Path(license_file) + if p.is_file(): + license_text = p.read_text() + except (OSError, ValueError): + pass + + body: dict[str, Any] = {"licenseFile": license_text, "mode": mode} + if release: + body["release"] = release + return _unwrap(client.put("/users/me/matlab-license", json=body)) + + +@_auto_client +def clearMatlabLicense( + *, + release: str = "", + client: _Client = None, +) -> dict[str, Any]: + """DELETE /users/me/matlab-license -- Remove a MATLAB BYOL registration. + + Without ``release``, fully clears the user's registration (releasing + the AWS ENI for dedicated mode). With ``release`` set, only that + release entry is removed from a dedicated registration; the MAC and + remaining releases stay intact. + + Server returns 204 on full clear or empty registration, 200 with + the remaining ``MatlabLicenseStatus`` when only one release was + removed. + + MATLAB equivalent: +cloud/+api/+users/clearMatlabLicense.m + """ + params: dict[str, str] | None = {"release": release} if release else None + return _unwrap(client.delete("/users/me/matlab-license", params=params)) diff --git a/src/ndi/cloud/auth.py b/src/ndi/cloud/auth.py index de5a440..1d446af 100644 --- a/src/ndi/cloud/auth.py +++ b/src/ndi/cloud/auth.py @@ -4,20 +4,24 @@ Provides JWT decoding, login/logout flows, and token management. MATLAB equivalents: - authenticate.m, login.m, logout.m - +internal/decodeJwt.m, getTokenExpiration.m, getActiveToken.m + authenticate.m, login.m, logout.m, testLogin.m + +internal/decodeJwt.m, getTokenExpiration.m, getActiveToken.m, + +internal/isTokenExpired.m """ from __future__ import annotations import base64 import json +import logging import os from datetime import datetime, timezone from .config import CloudConfig from .exceptions import CloudAuthError +logger = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # JWT helpers (no cryptographic verification — matches MATLAB behaviour) # --------------------------------------------------------------------------- @@ -79,25 +83,44 @@ def getTokenExpiration(token: str) -> datetime: return datetime.fromtimestamp(exp, tz=timezone.utc) -def verifyToken(token: str) -> bool: - """Check whether *token* is still valid (not expired). +def isTokenExpired(token: str) -> bool: + """Return True if *token* is expired, malformed, or empty. - Does **not** contact the server — only checks the ``exp`` claim. + Performs a local-only check by decoding the JWT's ``exp`` claim. + Does **not** contact the server. Mirrors the MATLAB helper + ``ndi.cloud.internal.isTokenExpired`` which was extracted from + ``authenticate.m`` so that callers can do a cheap pre-check + before issuing an authenticated request. + + MATLAB equivalent: +cloud/+internal/isTokenExpired.m """ if not token: - return False + return True try: expiration = getTokenExpiration(token) - return datetime.now(timezone.utc) < expiration except CloudAuthError: - return False + return True + return datetime.now(timezone.utc) >= expiration + + +def verifyToken(token: str) -> bool: + """Check whether *token* is still valid (not expired). + + Does **not** contact the server — only checks the ``exp`` claim. + Equivalent to ``not isTokenExpired(token)`` and kept for backward + compatibility. + """ + return not isTokenExpired(token) def getActiveToken(config: CloudConfig | None = None) -> tuple[str, str]: """Return ``(token, org_id)`` from *config* or environment. Raises: - CloudAuthError: If no valid token is available. + CloudAuthError: If no valid token is available, or if the + organization id is missing. Mirrors the MATLAB requirement + that the organization id be populated before the token can + be used for cached auth. """ if config is None: config = CloudConfig.from_env() @@ -105,12 +128,61 @@ def getActiveToken(config: CloudConfig | None = None) -> tuple[str, str]: if not config.token: raise CloudAuthError("No token available (NDI_CLOUD_TOKEN not set)") - if not verifyToken(config.token): + if isTokenExpired(config.token): raise CloudAuthError("Token is expired") + if not config.org_id: + raise CloudAuthError( + "Token is present but NDI_CLOUD_ORGANIZATION_ID is empty; " + "cached auth requires an organization id." + ) + return config.token, config.org_id +# --------------------------------------------------------------------------- +# Organization-id extraction (handles struct / list / dict shapes) +# --------------------------------------------------------------------------- + + +def _extract_first_organization_id(user: dict) -> str: + """Extract the first organization id from a login response's user. + + Mirrors MATLAB ``extractFirstOrganizationId``: accepts dict, list of + dicts, or a single dict; warns when multiple organizations are + present (we pick the first; explicit selection is not yet + implemented). + """ + if not isinstance(user, dict) or "organizations" not in user: + raise CloudAuthError("Login response did not include an organizations field.") + + orgs = user["organizations"] + org_id = "" + n_orgs = 0 + + if isinstance(orgs, dict) and "id" in orgs: + org_id = orgs.get("id", "") + n_orgs = 1 + elif isinstance(orgs, list) and orgs: + first = orgs[0] + if isinstance(first, dict) and "id" in first: + org_id = first.get("id", "") + n_orgs = len(orgs) + + if not org_id: + raise CloudAuthError("Could not extract an organization id from the login response.") + + if n_orgs > 1: + logger.warning( + "Login response contained %d organizations; using the first (%r). " + "Selection among multiple organizations is not yet supported.", + n_orgs, + org_id, + ) + + return str(org_id) + + # --------------------------------------------------------------------------- # Login / logout (require ``requests``) # --------------------------------------------------------------------------- @@ -168,14 +240,8 @@ def login( data = resp.json() token = data.get("token", "") - # Organisation ID from response - org_id = "" - user = data.get("user", {}) - orgs = user.get("organizations", {}) - if isinstance(orgs, dict): - org_id = orgs.get("id", "") - elif isinstance(orgs, list) and orgs: - org_id = orgs[0].get("id", "") + user = data.get("user", {}) or {} + org_id = _extract_first_organization_id(user) # Store in environment for other code to pick up os.environ["NDI_CLOUD_TOKEN"] = token @@ -226,7 +292,7 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: """Return an active token and organization ID, attempting login if needed. Priority (matching MATLAB ``authenticate.m``): - 1. Existing valid token in config/env. + 1. Existing valid token in config/env (local JWT exp pre-check). 2. Username + password from env → login. Args: @@ -242,8 +308,10 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: if config is None: config = CloudConfig.from_env() - # 1. Already have a valid token? - if config.token and verifyToken(config.token): + # 1. Already have a non-expired token AND an org id? Use it. + # Mirrors MATLAB isAuthenticated() which requires both token and + # organization_id to be present before short-circuiting. + if config.token and config.org_id and not isTokenExpired(config.token): return config.token, config.org_id # 2. Try env-var credentials @@ -259,6 +327,205 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: ) +# --------------------------------------------------------------------------- +# testLogin — non-mutating probe of the currently held token +# --------------------------------------------------------------------------- + + +def testLogin( + *, + user_name: str | None = None, + use_ui_login: bool = False, + verbose: bool = False, +) -> bool: + """Test whether the current process has a good NDI Cloud login. + + Returns True iff there is currently a valid login token in this + process from which a username (the JWT ``email`` claim) can be + extracted, AND that exact token is accepted by the server via a + direct ``GET /users/me`` with the token as the Bearer credential. + + The probe is deliberately issued as a raw HTTP request rather than + via :func:`ndi.cloud.api.users.me` (which routes through + :func:`authenticate` and could silently re-auth as a different user + mid-call). + + Order of operations: + + 1. Probe the currently active token. If it is valid and the + server accepts it, return True. + 2. Otherwise log out (clearing any stale token) and check for + silent credentials in the environment + (``NDI_CLOUD_USERNAME`` / ``NDI_CLOUD_PASSWORD``). + 3. If those env credentials are set, attempt a non-interactive + re-login via :func:`login`. Probe again. The UI login is + **never** shown when env credentials are present. + 4. Only if env credentials are empty AND ``use_ui_login`` is + True, would a UI login be shown — but Python has no GUI + equivalent, so this branch always returns False. + + Args: + user_name: If provided, the JWT in the active token must have + been issued for this email; otherwise the login is + considered not good even if the API call succeeds. + use_ui_login: Reserved for parity with MATLAB; always False in + effect for the Python implementation (no GUI). + verbose: If True, print step-by-step diagnostics to stderr. + + Returns: + True if the user has a valid login (and, when ``user_name`` is + provided, the token belongs to that user), False otherwise. + + MATLAB equivalent: +cloud/testLogin.m + """ + + def _log(msg: str) -> None: + if verbose: + print(f"[testLogin] {msg}") + + _log("Starting NDI Cloud login test.") + if user_name is None: + _log("No user_name specified; token-user check will be skipped.") + else: + _log(f"user_name specified: {user_name} (token must match).") + _log(f"use_ui_login = {use_ui_login}.") + + # Attempt 1: probe the currently active token. + _log("Attempt 1: probing the currently active token.") + if _probe(user_name, verbose): + _log("Attempt 1 succeeded. Returning True.") + return True + + # No good current token; clear stale state. + _log("Attempt 1 failed. Logging out to clear stale state.") + try: + logout() + except Exception as exc: # pragma: no cover - defensive + _log(f" logout raised: {exc}") + + # Attempt 2: silent re-auth via env credentials. + env_user = os.environ.get("NDI_CLOUD_USERNAME", "") + env_pass = os.environ.get("NDI_CLOUD_PASSWORD", "") + have_env_creds = bool(env_user) and bool(env_pass) + + if have_env_creds: + _log("Attempt 2: env credentials are set; attempting silent re-auth.") + if user_name is not None and env_user != user_name: + _log( + f" NDI_CLOUD_USERNAME ({env_user}) does not match requested " + f"user_name ({user_name}); skipping silent login." + ) + else: + try: + login(env_user, env_pass) + _log(" silent login completed.") + except Exception as exc: + _log(f" silent login raised: {exc}") + ok = _probe(user_name, verbose) + _log(f"Attempt 2 {'succeeded' if ok else 'failed'}. Returning {ok}.") + return ok + + # Attempt 3: env credentials are empty. Python has no GUI login, + # so when use_ui_login is True we still return False here. + _log("No env credentials available; Python has no UI login. " "Returning False.") + return False + + +def _probe(user_name: str | None, verbose: bool) -> bool: + """Direct GET /users/me probe of the current NDI_CLOUD_TOKEN. + + Implementation note: we deliberately do NOT go through + :func:`authenticate` or the CloudClient, both of which can silently + re-auth via env credentials. If that happened, the API call would + succeed and the probe would falsely report the original login as + good. Instead we read the raw token from the environment, do + local JWT validity checks, and send the request ourselves. + """ + + def _log(msg: str) -> None: + if verbose: + print(f"[testLogin] probe: {msg}") + + raw_token = os.environ.get("NDI_CLOUD_TOKEN", "") + if not raw_token: + _log("NDI_CLOUD_TOKEN is empty (no token in env). probe = False.") + return False + + try: + decoded = decodeJwt(raw_token) + except CloudAuthError as exc: + _log(f"decodeJwt failed: {exc}. probe = False.") + return False + + # Local expiration check. + if "exp" in decoded: + try: + exp_time = datetime.fromtimestamp(decoded["exp"], tz=timezone.utc) + except (TypeError, ValueError, OSError) as exc: + _log(f"could not parse exp claim: {exc}. probe = False.") + return False + if datetime.now(timezone.utc) >= exp_time: + _log(f"token expired at {exp_time.isoformat()}. probe = False.") + return False + + email_claim = decoded.get("email", "") + if not email_claim: + _log("token has no extractable username (no 'email' claim). probe = False.") + return False + + _log(f"token email = {email_claim}.") + if user_name is not None and email_claim != user_name: + _log(f"token email does NOT match user_name ({user_name}). probe = False.") + return False + + # Server-side verification with this exact token. + try: + import requests + except ImportError: + _log("requests not installed. probe = False.") + return False + + config = CloudConfig.from_env() + url = f"{config.api_url}/users/me" + _log(f"sending GET {url} with the current token.") + try: + resp = requests.get( + url, + headers={ + "Authorization": f"Bearer {raw_token}", + "Accept": "application/json", + }, + timeout=30, + ) + except requests.RequestException as exc: + _log(f"GET /users/me raised: {exc}") + return False + + if resp.status_code != 200: + _log(f"GET /users/me returned {resp.status_code}. probe = False.") + return False + + _log("GET /users/me returned 200 OK.") + + # Defense in depth: cross-check server email against JWT email. + try: + body = resp.json() + except ValueError: + body = {} + if isinstance(body, dict) and body.get("email"): + server_email = str(body["email"]) + if server_email.lower() != str(email_claim).lower(): + _log( + f"server email ({server_email}) does NOT match JWT email " + f"({email_claim}). probe = False." + ) + return False + _log("server email matches JWT email. probe = True.") + else: + _log("server response had no email field; trusting 200 status. probe = True.") + return True + + # --------------------------------------------------------------------------- # Account management (require ``requests``) # --------------------------------------------------------------------------- diff --git a/src/ndi/cloud/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/ndi_matlab_python_bridge.yaml index 1f73ce9..8b956ff 100644 --- a/src/ndi/cloud/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/ndi_matlab_python_bridge.yaml @@ -2,7 +2,7 @@ # The Primary Contract for the ndi.cloud root namespace. # # Covers: auth.py, config.py, client.py, exceptions.py, orchestration.py, -# download.py, upload.py, filehandler.py, internal.py +# download.py, upload.py, filehandler.py, internal.py, profile.py project_metadata: bridge_version: "1.1" @@ -80,7 +80,7 @@ functions: # --- auth.py functions --- - name: authenticate matlab_path: "+ndi/+cloud/authenticate.m" - matlab_last_sync_hash: "80092b7c" + matlab_last_sync_hash: "2566fe4d" python_path: "ndi/cloud/auth.py" input_arguments: - name: config @@ -93,8 +93,47 @@ functions: - name: organizationID type_python: "str" decision_log: > - MATLAB authenticate returns [token, organizationID]. - Python now returns (token, org_id) tuple to match. + Synchronized with MATLAB main as of 2026-05-11 (matlab HEAD + 2566fe4d). Now requires both token AND organization_id to be + populated before treating a cached login as valid (mirrors + commit 5c4352eeac). The local JWT exp pre-check is delegated to + isTokenExpired (extracted in commit 9562eb28e7). Organization-id + extraction (auth._extract_first_organization_id) accepts dict, + list-of-dicts, or single dict, and warns when multiple + organizations are present. + + - name: testLogin + matlab_path: "+ndi/+cloud/testLogin.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/auth.py" + input_arguments: + - name: user_name + type_matlab: "string" + type_python: "str | None" + default: "None" + - name: use_ui_login + type_matlab: "logical" + type_python: "bool" + default: "False" + - name: verbose + type_matlab: "logical" + type_python: "bool" + default: "False" + output_arguments: + - name: is_good + type_python: "bool" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d). + Three-attempt cascade: (1) probe NDI_CLOUD_TOKEN via direct GET + /users/me bypassing wrappers (to avoid silent re-auth swapping + the token mid-call); (2) if NDI_CLOUD_USERNAME / NDI_CLOUD_PASSWORD + are set, do a silent login and re-probe; (3) if env credentials + are absent, return False (Python has no GUI login -- use_ui_login + is accepted for parity but never escalates to UI). Local JWT + validity check is performed before any HTTP call. Verbose mode + prints step-by-step diagnostics. UI step (MATLAB ndi.cloud.uilogin) + is intentionally omitted from the Python port; that is part of the + profileEditor GUI workstream. - name: login matlab_path: "+ndi/+cloud/+api/+auth/login.m" @@ -226,6 +265,25 @@ functions: MATLAB places this in +internal; Python places it in auth.py. Name exact match. + - name: isTokenExpired + matlab_path: "+ndi/+cloud/+internal/isTokenExpired.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/auth.py" + input_arguments: + - name: token + type_matlab: "char" + type_python: "str" + output_arguments: + - name: expired + type_python: "bool" + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d, + original commit 716442b940). Local-only JWT exp check; True for + empty / malformed / expired tokens. MATLAB places this in + +internal; Python places it in auth.py alongside the other JWT + helpers. The pre-existing Python verifyToken is now defined in + terms of `not isTokenExpired`. + - name: verifyToken matlab_path: "N/A" python_path: "ndi/cloud/auth.py" @@ -239,7 +297,7 @@ functions: - name: getActiveToken matlab_path: "+ndi/+cloud/+internal/getActiveToken.m" - matlab_last_sync_hash: "7ab96514" + matlab_last_sync_hash: "2566fe4d" python_path: "ndi/cloud/auth.py" input_arguments: - name: config @@ -251,8 +309,10 @@ functions: - name: org_id type_python: "str" decision_log: > - MATLAB places this in +internal; Python places it in auth.py. - Name exact match. + Updated 2026-05-11 to require a non-empty organization id + (matches MATLAB commit 5c4352eeac which made org id required for + cached auth). MATLAB places this in +internal; Python places it + in auth.py. Name exact match. # --- orchestration.py functions --- - name: downloadDataset @@ -400,9 +460,6 @@ functions: Snake_case is acceptable for Python-only functions. # --- download.py functions --- - # NOTE: dataset (old one-by-one pipeline) removed from both MATLAB and - # Python. Superseded by downloadDocumentCollection + orchestration.downloadDataset. - - name: downloadDocumentCollection matlab_path: "+ndi/+cloud/+download/downloadDocumentCollection.m" matlab_last_sync_hash: "fbf6d337" @@ -494,9 +551,6 @@ functions: it is used by the modern orchestration layer (downloadDataset, syncDataset). - # NOTE: datasetDocuments (old one-by-one pipeline) removed from both - # MATLAB and Python. Superseded by downloadDocumentCollection. - - name: downloadGenericFiles matlab_path: "+ndi/+cloud/+download/downloadGenericFiles.m" matlab_last_sync_hash: "cafbfe74" @@ -535,11 +589,6 @@ functions: type_python: "dict[str, Any]" decision_log: "Exact match." - # NOTE: setFileInfo (old pipeline) removed from both MATLAB and Python. - # File-info patching now lives in the sync layer: - # updateFileInfoForLocalFiles → ndi.cloud.sync.internal - # updateFileInfoForRemoteFiles → ndi.cloud.sync.internal - - name: structsToNdiDocuments matlab_path: "+ndi/+cloud/+download/+internal/structsToNdiDocuments.m" matlab_last_sync_hash: "211a705e" @@ -555,8 +604,6 @@ functions: MATLAB places this in +download/+internal; Python places it in download.py. Exact name match. - # NOTE: downloadFullDataset alias removed along with download.dataset. - # --- upload.py functions --- - name: uploadDocumentCollection matlab_path: "+ndi/+cloud/+upload/uploadDocumentCollection.m" @@ -943,6 +990,33 @@ functions: type_python: "list[dict[str, Any]]" decision_log: "Exact match." + # --- profile.py (singleton manager for cloud login profiles) --- + - name: profile + matlab_path: "+ndi/+cloud/profile.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/cloud/profile.py" + input_arguments: [] + output_arguments: [] + decision_log: > + Ported from MATLAB main as of 2026-05-11 (matlab HEAD 2566fe4d, + commit 970b6854b1). MATLAB exposes ndi.cloud.profile as a singleton + classdef with static methods (add, remove, getCurrent, setCurrent, + getDefault, setDefault, getPassword, setPassword, getStage, + setStage, switchProfile, etc.). Python ports the same surface as + module-level functions in ndi.cloud.profile, backed by a + _ProfileSingleton dataclass. Backends: keyring (preferred, + mirrors MATLAB's "vault"), aes (mirrors MATLAB's AES file + backend), or memory (test-only). Profile metadata persists to + ~/.ndi/NDI_Cloud_Profiles.json; AES ciphertext (when used) goes + to NDI_Cloud_Secrets.json in the same directory. Python uses + Python-idiomatic snake_case for the public functions + (list_profiles, get_current, set_current, set_default, + switch_profile, use_backend) since these are convenience helpers, + not strict API mirrors; ProfileEntry field names (UID, Nickname, + Email, Stage, PasswordSecret) match MATLAB exactly so the on-disk + JSON is interchangeable. GUI editor (MATLAB ndi.gui.profileEditor) + is intentionally NOT ported -- separate workstream. + # ============================================================================= # Not-yet-ported MATLAB functions # ============================================================================= @@ -998,7 +1072,11 @@ not_yet_ported: matlab_path: "+ndi/+cloud/uilogin.m" matlab_last_sync_hash: "80092b7c" status: not_applicable - decision_log: "MATLAB GUI login. No Python equivalent needed." + decision_log: > + MATLAB GUI login. No Python equivalent; testLogin() in Python + simply returns False when env credentials are absent (since a + Python CLI session has no UI). The MATLAB profileEditor GUI is a + separate workstream. - name: zip_documents_for_upload matlab_path: "+ndi/+cloud/+upload/+internal/zip_documents_for_upload.m" @@ -1006,4 +1084,4 @@ not_yet_ported: status: not_applicable decision_log: > MATLAB internal helper. Python uses zipForUpload which handles - this internally with zipfile module. + this internally with zipfile module. \ No newline at end of file diff --git a/src/ndi/cloud/profile.py b/src/ndi/cloud/profile.py new file mode 100644 index 0000000..9f5eaa3 --- /dev/null +++ b/src/ndi/cloud/profile.py @@ -0,0 +1,506 @@ +""" +ndi.cloud.profile - Singleton manager for NDI Cloud user profiles. + +This is the Python port of MATLAB ``ndi.cloud.profile``. It keeps a +list of NDI Cloud login profiles for the current OS user. Each profile +carries a ``Nickname``, an ``Email``, an auto-generated ``UID``, and a +``Stage`` (``'prod'`` or ``'dev'``). Passwords are not stored in the +profile JSON; instead each profile points at a secret keyed by +``'NDI Cloud ' + UID`` in a pluggable backend. + +Backends, chosen automatically on first use: + + keyring -- the OS-native credential store via the ``keyring`` + package. Preferred when available. Equivalent to + MATLAB's "vault" backend. + aes -- AES-128/CBC encrypted file in the user's prefdir, + used when ``keyring`` is not installed. The key is + derived from SHA-256([hostname username 'NDI Cloud']) + so the file is reproducible only on the machine that + wrote it. + memory -- in-memory dict. Reserved for tests; use + ``ndi.cloud.profile.use_backend('memory')`` to opt in. + +Current vs default profile +-------------------------- +The class distinguishes between two notions of "selected": + + current_uid - the active profile for THIS Python process. + Held in memory only; never persisted. + default_uid - the user's preferred profile, persisted to the JSON + file. At construction the singleton copies a valid + ``default_uid`` into ``current_uid``. + +MATLAB equivalent: +cloud/profile.m +""" + +from __future__ import annotations + +import getpass +import hashlib +import json +import logging +import os +import secrets +import socket +import tempfile +import uuid +from base64 import b64decode, b64encode +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Literal + +logger = logging.getLogger(__name__) + + +_BackendName = Literal["keyring", "aes", "memory"] +_SECRET_KEY_PREFIX = "NDI Cloud " + + +# --------------------------------------------------------------------------- +# Profile entry +# --------------------------------------------------------------------------- + + +@dataclass +class ProfileEntry: + """One entry in the profile list.""" + + UID: str = "" + Nickname: str = "" + Email: str = "" + Stage: str = "prod" + PasswordSecret: str = "" + + +def _prefdir() -> Path: + """Return the directory where profile state is persisted. + + Honours ``NDI_PREFDIR`` if set, otherwise uses + ``~/.ndi`` (created if absent), falling back to the system temp + directory if the home dir is unwritable. + """ + override = os.environ.get("NDI_PREFDIR", "") + if override: + return Path(override) + try: + d = Path.home() / ".ndi" + d.mkdir(parents=True, exist_ok=True) + return d + except OSError: + return Path(tempfile.gettempdir()) + + +# --------------------------------------------------------------------------- +# AES helpers (used when no OS keyring is available) +# --------------------------------------------------------------------------- + + +def _aes_key_bytes() -> bytes: + try: + host = socket.gethostname() + except OSError: # pragma: no cover - extreme defensive + host = "localhost" + try: + user = getpass.getuser() + except Exception: # pragma: no cover - extreme defensive + user = "unknown" + seed = f"{host} {user} NDI Cloud".encode() + return hashlib.sha256(seed).digest()[:16] + + +def _aes_encrypt(value: str) -> dict[str, str]: + """Encrypt *value* with AES-128/CBC and return iv+ciphertext (base64).""" + from cryptography.hazmat.primitives import padding + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + key = _aes_key_bytes() + iv = secrets.token_bytes(16) + padder = padding.PKCS7(128).padder() + padded = padder.update(value.encode("utf-8")) + padder.finalize() + enc = Cipher(algorithms.AES(key), modes.CBC(iv)).encryptor() + ct = enc.update(padded) + enc.finalize() + return { + "iv": b64encode(iv).decode("ascii"), + "ciphertext": b64encode(ct).decode("ascii"), + } + + +def _aes_decrypt(entry: dict) -> str: + from cryptography.hazmat.primitives import padding + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + key = _aes_key_bytes() + iv = b64decode(entry["iv"]) + ct = b64decode(entry["ciphertext"]) + dec = Cipher(algorithms.AES(key), modes.CBC(iv)).decryptor() + padded = dec.update(ct) + dec.finalize() + unpadder = padding.PKCS7(128).unpadder() + plain = unpadder.update(padded) + unpadder.finalize() + return plain.decode("utf-8") + + +def _read_secrets_file(filename: Path) -> dict: + if not filename.is_file(): + return {} + try: + return json.loads(filename.read_text()) + except (ValueError, OSError): + return {} + + +def _write_secrets_file(filename: Path, payload: dict) -> None: + filename.write_text(json.dumps(payload, indent=2)) + + +def _safe_field(name: str) -> str: + """Map a secret key to a JSON-safe field name.""" + return name.replace(" ", "_").replace(":", "_") + + +# --------------------------------------------------------------------------- +# Backend detection +# --------------------------------------------------------------------------- + + +def _detect_backend() -> _BackendName: + try: + import keyring # noqa: F401 + except ImportError: + try: + from cryptography.hazmat.primitives.ciphers import Cipher # noqa: F401 + except ImportError: + logger.warning( + "Neither 'keyring' nor 'cryptography' is installed; " + "ndi.cloud.profile will fall back to the in-memory backend " + "which does NOT persist secrets to disk." + ) + return "memory" + return "aes" + return "keyring" + + +# --------------------------------------------------------------------------- +# Singleton +# --------------------------------------------------------------------------- + + +@dataclass +class _ProfileSingleton: + profiles: list[ProfileEntry] = field(default_factory=list) + current_uid: str = "" + default_uid: str = "" + backend: _BackendName = "memory" + _memory_store: dict[str, str] = field(default_factory=dict) + + # ------------- filesystem paths ------------- + + @property + def filename(self) -> Path: + return _prefdir() / "NDI_Cloud_Profiles.json" + + @property + def secrets_filename(self) -> Path: + return _prefdir() / "NDI_Cloud_Secrets.json" + + # ------------- disk I/O ------------- + + def _load_from_disk(self) -> None: + if not self.filename.is_file(): + return + try: + data = json.loads(self.filename.read_text()) + except (ValueError, OSError) as exc: + logger.warning("Could not load cloud profiles from %s: %s", self.filename, exc) + return + raw = data.get("Profiles") or [] + self.profiles = [] + for item in raw: + if not isinstance(item, dict): + continue + entry = ProfileEntry( + UID=str(item.get("UID", "")), + Nickname=str(item.get("Nickname", "")), + Email=str(item.get("Email", "")), + Stage=str(item.get("Stage", "prod")), + PasswordSecret=str(item.get("PasswordSecret", "")), + ) + if not entry.PasswordSecret and entry.UID: + entry.PasswordSecret = _SECRET_KEY_PREFIX + entry.UID + self.profiles.append(entry) + self.default_uid = str(data.get("DefaultUID", "")) + + def _save_to_disk(self) -> None: + payload = { + "Profiles": [asdict(p) for p in self.profiles], + "DefaultUID": self.default_uid, + } + try: + self.filename.write_text(json.dumps(payload, indent=2)) + except OSError as exc: + logger.warning("Could not save cloud profiles to %s: %s", self.filename, exc) + + def _adopt_default_as_current(self) -> None: + if not self.default_uid or not self.profiles: + return + if any(p.UID == self.default_uid for p in self.profiles): + self.current_uid = self.default_uid + + # ------------- lookup ------------- + + def _find_index(self, uid: str) -> int: + for i, p in enumerate(self.profiles): + if p.UID == uid: + return i + raise KeyError(f'Unknown profile UID "{uid}".') + + # ------------- secrets backend ------------- + + def _set_secret(self, key: str, value: str) -> None: + if self.backend == "keyring": + import keyring + + keyring.set_password("ndi-cloud", key, value) + elif self.backend == "aes": + store = _read_secrets_file(self.secrets_filename) + store[_safe_field(key)] = _aes_encrypt(value) + _write_secrets_file(self.secrets_filename, store) + else: # memory + self._memory_store[key] = value + + def _get_secret(self, key: str) -> str: + if self.backend == "keyring": + import keyring + + value = keyring.get_password("ndi-cloud", key) + if value is None: + raise KeyError(f'No secret stored for "{key}".') + return value + if self.backend == "aes": + store = _read_secrets_file(self.secrets_filename) + entry = store.get(_safe_field(key)) + if entry is None: + raise KeyError(f'No secret stored for "{key}".') + return _aes_decrypt(entry) + if key not in self._memory_store: + raise KeyError(f'No secret stored for "{key}".') + return self._memory_store[key] + + def _remove_secret(self, key: str) -> None: + if self.backend == "keyring": + import keyring + + try: + keyring.delete_password("ndi-cloud", key) + except Exception: # noqa: BLE001 - keyring raises a family of errors + pass + elif self.backend == "aes": + store = _read_secrets_file(self.secrets_filename) + store.pop(_safe_field(key), None) + _write_secrets_file(self.secrets_filename, store) + else: + self._memory_store.pop(key, None) + + +_singleton: _ProfileSingleton | None = None + + +def _get_singleton() -> _ProfileSingleton: + global _singleton + if _singleton is None: + obj = _ProfileSingleton(backend=_detect_backend()) + obj._load_from_disk() + obj._adopt_default_as_current() + _singleton = obj + return _singleton + + +# --------------------------------------------------------------------------- +# Public API (mirrors MATLAB static methods) +# --------------------------------------------------------------------------- + + +def list_profiles() -> list[ProfileEntry]: + """Return a shallow copy of the profile list.""" + return list(_get_singleton().profiles) + + +def get(uid: str) -> ProfileEntry: + """Return the profile entry for *uid*.""" + obj = _get_singleton() + return obj.profiles[obj._find_index(uid)] + + +def add(nickname: str, email: str, password: str) -> str: + """Create a new profile, store its password, and return the new UID.""" + obj = _get_singleton() + uid = uuid.uuid4().hex + secret_key = _SECRET_KEY_PREFIX + uid + entry = ProfileEntry( + UID=uid, + Nickname=nickname, + Email=email, + Stage="prod", + PasswordSecret=secret_key, + ) + obj.profiles.append(entry) + obj._set_secret(secret_key, password) + obj._save_to_disk() + return uid + + +def remove(uid: str) -> None: + """Delete a profile and its stored secret.""" + obj = _get_singleton() + idx = obj._find_index(uid) + secret_key = obj.profiles[idx].PasswordSecret + obj._remove_secret(secret_key) + del obj.profiles[idx] + if obj.current_uid == uid: + obj.current_uid = "" + if obj.default_uid == uid: + obj.default_uid = "" + obj._save_to_disk() + + +def get_current() -> ProfileEntry | None: + """Return the active profile for this session, or None.""" + obj = _get_singleton() + if not obj.current_uid: + return None + try: + return get(obj.current_uid) + except KeyError: + return None + + +def set_current(uid: str) -> None: + """Set the current profile for this session (in memory only).""" + obj = _get_singleton() + obj._find_index(uid) # validates existence + obj.current_uid = uid + + +def get_default() -> ProfileEntry | None: + """Return the persisted default profile, or None.""" + obj = _get_singleton() + if not obj.default_uid: + return None + try: + return get(obj.default_uid) + except KeyError: + return None + + +def set_default(uid: str) -> None: + """Persist *uid* as the default profile.""" + obj = _get_singleton() + obj._find_index(uid) + obj.default_uid = uid + obj._save_to_disk() + + +def clear_default() -> None: + """Forget any persisted default.""" + obj = _get_singleton() + obj.default_uid = "" + obj._save_to_disk() + + +def get_password(uid: str) -> str: + """Retrieve the stored password for *uid*.""" + obj = _get_singleton() + idx = obj._find_index(uid) + return obj._get_secret(obj.profiles[idx].PasswordSecret) + + +def set_password(uid: str, password: str) -> None: + """Update a profile's stored password.""" + obj = _get_singleton() + idx = obj._find_index(uid) + obj._set_secret(obj.profiles[idx].PasswordSecret, password) + + +def get_stage(uid: str) -> str: + """Return the profile's Stage.""" + obj = _get_singleton() + return obj.profiles[obj._find_index(uid)].Stage + + +def set_stage(uid: str, stage: str) -> None: + """Set the profile's Stage to ``'prod'`` or ``'dev'``.""" + if stage not in ("prod", "dev"): + raise ValueError("stage must be 'prod' or 'dev'") + obj = _get_singleton() + idx = obj._find_index(uid) + obj.profiles[idx].Stage = stage + obj._save_to_disk() + + +def switch_profile(uid: str) -> None: + """Make *uid* the active profile and reconfigure env vars. + + Calls :func:`ndi.cloud.logout`, then sets: + + CLOUD_API_ENVIRONMENT -> profile.Stage + NDI_CLOUD_USERNAME -> profile.Email + NDI_CLOUD_PASSWORD -> get_password(uid) + + Marks *uid* as the current profile (in memory only -- does not + change the persisted default). + """ + obj = _get_singleton() + prof = obj.profiles[obj._find_index(uid)] + try: + from .auth import logout + + logout() + except Exception as exc: # noqa: BLE001 - parity with MATLAB warning + logger.warning("logout failed during switch_profile: %s", exc) + + os.environ["CLOUD_API_ENVIRONMENT"] = prof.Stage + os.environ["NDI_CLOUD_USERNAME"] = prof.Email + os.environ["NDI_CLOUD_PASSWORD"] = obj._get_secret(prof.PasswordSecret) + obj.current_uid = uid + + +def filename() -> Path: + """Return the JSON profile-list path.""" + return _get_singleton().filename + + +def secrets_filename() -> Path: + """Return the AES secrets file path.""" + return _get_singleton().secrets_filename + + +def backend() -> _BackendName: + """Return the active secrets backend (``'keyring'``/``'aes'``/``'memory'``).""" + return _get_singleton().backend + + +def use_backend(name: _BackendName) -> None: + """Force a backend (test hook). ``name`` must be one of + ``'keyring'``, ``'aes'``, ``'memory'``.""" + if name not in ("keyring", "aes", "memory"): + raise ValueError("backend must be 'keyring', 'aes', or 'memory'") + _get_singleton().backend = name + + +def reload() -> None: + """Re-read profiles and default from disk.""" + obj = _get_singleton() + obj.profiles = [] + obj.current_uid = "" + obj.default_uid = "" + obj._load_from_disk() + obj._adopt_default_as_current() + + +def reset() -> None: + """Clear the in-memory singleton state. Does NOT touch disk.""" + obj = _get_singleton() + obj.profiles = [] + obj.current_uid = "" + obj.default_uid = "" + obj._memory_store = {} diff --git a/src/ndi/daq/metadatareader/__init__.py b/src/ndi/daq/metadatareader/__init__.py index ef54166..fc47145 100644 --- a/src/ndi/daq/metadatareader/__init__.py +++ b/src/ndi/daq/metadatareader/__init__.py @@ -306,11 +306,13 @@ def __hash__(self) -> int: # Import subclass readers from .newstim_stims import ndi_daq_metadatareader_NewStimStims from .nielsenlab_stims import ndi_daq_metadatareader_NielsenLabStims +from .rayolab_stims import ndi_daq_metadatareader_RayoLabStims from .vhaudreybpod_stims import ndi_daq_metadatareader_VHAudreyBPod __all__ = [ "ndi_daq_metadatareader", "ndi_daq_metadatareader_NewStimStims", "ndi_daq_metadatareader_NielsenLabStims", + "ndi_daq_metadatareader_RayoLabStims", "ndi_daq_metadatareader_VHAudreyBPod", ] diff --git a/src/ndi/daq/metadatareader/ndi_matlab_python_bridge.yaml b/src/ndi/daq/metadatareader/ndi_matlab_python_bridge.yaml index b568ebf..5575719 100644 --- a/src/ndi/daq/metadatareader/ndi_matlab_python_bridge.yaml +++ b/src/ndi/daq/metadatareader/ndi_matlab_python_bridge.yaml @@ -308,3 +308,63 @@ classes: - name: parameters type_python: "list[dict[str, Any]]" decision_log: "Python-specific. Reads Analyzer from .mat file." + + # ========================================================================= + # ndi.daq.metadatareader.RayoLabStims + # ========================================================================= + - name: RayoLabStims + type: class + matlab_path: "+ndi/+daq/+metadatareader/RayoLabStims.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/daq/metadatareader/rayolab_stims.py" + python_class: "ndi_daq_metadatareader_RayoLabStims" + inherits: "ndi.daq.metadatareader" + + methods: + - name: RayoLabStims + kind: constructor + input_arguments: + - name: tsv_pattern + type_python: "str" + default: "''" + - name: identifier + type_python: "str | None" + default: "None" + - name: session + type_python: "Any | None" + default: "None" + - name: document + type_python: "Any | None" + default: "None" + output_arguments: + - name: reader_obj + type_python: "ndi_daq_metadatareader_RayoLabStims" + decision_log: > + Exact match. Forwards all arguments to base class. The + tsv_pattern is stored but not consulted; parameters are + constant. + + - name: readmetadata + input_arguments: + - name: epochfiles + type_matlab: "cell array of char" + type_python: "list[str]" + output_arguments: + - name: parameters + type_python: "list[dict[str, Any]]" + decision_log: > + Trivial constant reader. Ignores epochfiles and always + returns [{"stimid": 1}], matching the mk2 marker channel + stimulus id emitted by the RayoLab intan-series stimulator. + + - name: readmetadatafromfile + input_arguments: + - name: filepath + type_matlab: "char" + type_python: "str" + output_arguments: + - name: parameters + type_python: "list[dict[str, Any]]" + decision_log: > + Trivial constant reader. Ignores filepath and always + returns [{"stimid": 1}], matching the MATLAB class. diff --git a/src/ndi/daq/metadatareader/rayolab_stims.py b/src/ndi/daq/metadatareader/rayolab_stims.py new file mode 100644 index 0000000..bca8d8d --- /dev/null +++ b/src/ndi/daq/metadatareader/rayolab_stims.py @@ -0,0 +1,84 @@ +"""ndi.daq.metadatareader.RayoLabStims - Trivial RayoLab stimulator metadata reader. + +The RayoLab stimulator emits a single stimulus type whose only parameter +is its stimulus id, which is always 1. This metadata reader does not read +any per-stimulus content from disk; it returns one constant parameter +set:: + + parameters[0] = {"stimid": 1} + +The single entry is keyed at index 1 to match the stimulus id reported on +the mk2 marker channel by +``ndi.setup.daq.reader.mfdaq.stimulus.rayolab_intanseries`` (MATLAB). + +The constructor accepts the same arguments as +:class:`ndi_daq_metadatareader` (typically a filename regular expression +identifying the metadata file inside an epoch's file list); the pattern +is stored but not consulted, since the parameters are constant. + +MATLAB equivalent: ``src/ndi/+ndi/+daq/+metadatareader/RayoLabStims.m`` +""" + +from __future__ import annotations + +from typing import Any + +from ..metadatareader import ndi_daq_metadatareader + + +class ndi_daq_metadatareader_RayoLabStims(ndi_daq_metadatareader): + """Constant metadata reader for the RayoLab stimulator. + + Always returns a single-element parameter list ``[{"stimid": 1}]``, + regardless of which epoch files are passed in. + + Example: + >>> reader = ndi_daq_metadatareader_RayoLabStims() + >>> reader.readmetadata(["some_epoch_file.rhd"]) + [{'stimid': 1}] + """ + + def __init__( + self, + tsv_pattern: str = "", + identifier: str | None = None, + session: Any | None = None, + document: Any | None = None, + ): + super().__init__( + tsv_pattern=tsv_pattern, + identifier=identifier, + session=session, + document=document, + ) + + def readmetadata( + self, + epochfiles: list[str], + ) -> list[dict[str, Any]]: + """Return the constant RayoLab parameter set. + + Args: + epochfiles: Ignored. + + Returns: + ``[{"stimid": 1}]`` + """ + return [{"stimid": 1}] + + def readmetadatafromfile( + self, + filepath: str, + ) -> list[dict[str, Any]]: + """Return the constant RayoLab parameter set. + + Args: + filepath: Ignored. + + Returns: + ``[{"stimid": 1}]`` + """ + return [{"stimid": 1}] + + def __repr__(self) -> str: + return f"ndi_daq_metadatareader_RayoLabStims(id='{self.id[:8]}...')" diff --git a/src/ndi/file/navigator/__init__.py b/src/ndi/file/navigator/__init__.py index 267c653..7ff4e9f 100644 --- a/src/ndi/file/navigator/__init__.py +++ b/src/ndi/file/navigator/__init__.py @@ -16,6 +16,7 @@ from ...ido import ndi_ido from ...time import NO_TIME, ndi_time_clocktype +from ...util.matlab_regex import matlab_to_python_regex def _to_matlab_cell_str(items: list[str]) -> str: @@ -121,7 +122,7 @@ def find_file_groups( break # Try regex pattern try: - if re.search(pattern, filename): + if re.search(matlab_to_python_regex(pattern), filename): matched_files.append((pat_idx, f)) break except re.error: @@ -649,7 +650,7 @@ def epochprobemapfilename( if fnmatch.fnmatch(filename, pattern): return f try: - if re.search(pattern, filename): + if re.search(matlab_to_python_regex(pattern), filename): return f except re.error: pass @@ -715,7 +716,7 @@ def getepochprobemap( if fnmatch.fnmatch(fname, glob_pat): return self._load_epochprobemap_file(f) try: - if re.search(pat, fname): + if re.search(matlab_to_python_regex(pat), fname): return self._load_epochprobemap_file(f) except re.error: pass diff --git a/src/ndi/file/navigator/epochdir.py b/src/ndi/file/navigator/epochdir.py index 28ae344..3ec5626 100644 --- a/src/ndi/file/navigator/epochdir.py +++ b/src/ndi/file/navigator/epochdir.py @@ -142,6 +142,8 @@ def _match_files_in_dir( import fnmatch import re + from ...util.matlab_regex import matlab_to_python_regex + matched = [] try: files = [f for f in directory.iterdir() if f.is_file()] @@ -156,7 +158,7 @@ def _match_files_in_dir( matched.append(str(f)) break try: - if re.search(pattern, f.name): + if re.search(matlab_to_python_regex(pattern), f.name): matched.append(str(f)) break except re.error: diff --git a/src/ndi/file/navigator/rhd_series.py b/src/ndi/file/navigator/rhd_series.py new file mode 100644 index 0000000..5364ee8 --- /dev/null +++ b/src/ndi/file/navigator/rhd_series.py @@ -0,0 +1,241 @@ +""" +ndi.file.navigator.rhd_series - File navigator for prefix-grouped .rhd recordings. + +Groups .rhd files that share a common prefix but differ in a trailing variable +portion (typically a YYYYMMDDHHMMSS.msec timestamp) into a single epoch. +Because the Intan reader can discover the remaining files in a series from +the first file alone, only the lexicographically earliest match in each +prefix group is returned. Ancillary files (e.g. an epochprobemap) that +share the same prefix are matched through the standard '#' substitution +syntax used by ndi.file.navigator. + +This navigator is for "flat" sessions in which all .rhd files of all epochs +live directly in the session directory. Use +ndi.file.navigator.rhd_series_epochdir for sessions in which each epoch +lives in its own subdirectory. + +MATLAB equivalent: +ndi/+file/+navigator/rhd_series.m +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + +from ...util.matlab_regex import matlab_to_python_regex +from . import ndi_file_navigator + + +class ndi_file_navigator_rhd_series(ndi_file_navigator): + """ + Navigator for prefix-grouped .rhd recordings in a flat session directory. + + FILEPARAMETERS + The filematch list is interpreted as follows: + + patterns[0] - Series pattern. Must contain exactly one '#'. + The '#' captures the per-epoch prefix; the rest of the pattern + is a regular expression matching the variable portion of the + filename. Files matching this pattern are grouped by the + captured prefix and each unique prefix becomes one epoch. + Within a group only the lexicographically earliest filename + is kept (which is the chronologically earliest when + timestamps are zero-padded). + + patterns[1:] - Ancillary patterns. In each pattern '#' is replaced + by the literal (regex-escaped) prefix of the current epoch + and the result is matched as a regular expression against the + filenames in the session directory. The lexicographically + earliest match is appended to the epoch's file list. If any + ancillary pattern produces no match the epoch is skipped. + + The epoch identifier returned by epochid is the prefix captured by + the series pattern. + + Example: + >>> nav = ndi_file_navigator_rhd_series( + ... session, + ... [r'#_\\d{14}\\.\\d+\\.rhd\\>', r'#\\.epochprobemap\\.ndi\\>'], + ... ) + """ + + NDI_FILENAVIGATOR_CLASS = "ndi.file.navigator.rhd_series" + + def epochid( + self, + epoch_number: int, + epochfiles: list[str] | None = None, + ) -> str: + """ + Return the epoch identifier for an epoch. + + MATLAB equivalent: ndi.file.navigator.rhd_series/epochid + + Returns the prefix captured by the series pattern from the first + file of the epoch. If the epoch's files are ingested, the + inherited ingested-file identifier is returned instead. + + Args: + epoch_number: epoch number (1-indexed) + epochfiles: optional file list (fetched if not provided) + + Returns: + Epoch identifier string. + """ + if epochfiles is None: + epochfiles = self.getepochfiles_number(epoch_number) + + if self.isingested(epochfiles): + return self.ingestedfiles_epochid(epochfiles) + + patterns = self._normalize_patterns(self._fileparameters.get("filematch", [])) + basename = os.path.basename(epochfiles[0]) + eid = self._extract_prefix(basename, patterns[0]) if patterns else "" + if not eid: + # Fall back to the file stem (no extension), matching MATLAB + # fileparts behavior. + eid = Path(basename).stem + return eid + + def selectfilegroups_disk(self) -> list[list[str]]: + """ + Return groups of files that comprise epochs. + + MATLAB equivalent: ndi.file.navigator.rhd_series/selectfilegroups_disk + + Inspects the session directory and returns one list per epoch, + each containing the absolute path of the first .rhd file in the + prefix group followed by any ancillary files matched by the + remaining filematch patterns. + + Returns: + List of epoch file groups. + """ + try: + base_path = self.path() + except ValueError: + return [] + + if not os.path.isdir(base_path): + return [] + + patterns = self._normalize_patterns(self._fileparameters.get("filematch", [])) + if not patterns: + return [] + + return self._group_directory(base_path, patterns) + + # ------------------------------------------------------------------ + # Static helpers + # ------------------------------------------------------------------ + + @staticmethod + def _normalize_patterns(filematch: str | list[str]) -> list[str]: + """Coerce filematch into a list of pattern strings. + + MATLAB equivalent: ndi.file.navigator.rhd_series.normalizePatterns + """ + if isinstance(filematch, str): + return [filematch] + return list(filematch) + + @staticmethod + def _extract_prefix(filename: str, series_pattern: str) -> str: + """Return the substring captured by '#' in series_pattern. + + MATLAB equivalent: ndi.file.navigator.rhd_series.extractPrefix + + Replaces the '#' in series_pattern with a non-greedy capture + group, anchors the result, and returns the captured substring + or '' if the filename does not match. + """ + rx = "^" + matlab_to_python_regex(series_pattern).replace("#", "(.+?)") + "$" + m = re.match(rx, filename) + if not m: + return "" + return m.group(1) + + @staticmethod + def _group_directory( + directory: str, + patterns: list[str], + ) -> list[list[str]]: + """Build epoch file groups from one directory. + + MATLAB equivalent: ndi.file.navigator.rhd_series.groupDirectory + + Applies the rhd_series matching rules to the files in directory + and returns a list of epoch file lists. patterns[0] is the + series pattern and patterns[1:] are ancillary patterns. + + Files whose basename begins with '.' (e.g. macOS resource forks + like '._foo.rhd' or '.DS_Store') are ignored, matching the + convention used by the base ndi.file.navigator. + """ + groups: list[list[str]] = [] + try: + entries = os.listdir(directory) + except (FileNotFoundError, NotADirectoryError, PermissionError): + return groups + + names = [ + n + for n in entries + if not n.startswith(".") and os.path.isfile(os.path.join(directory, n)) + ] + if not names: + return groups + + series_regex = "^" + matlab_to_python_regex(patterns[0]).replace("#", "(.+?)") + "$" + series_re = re.compile(series_regex) + + # Tokenize: keep names that match and capture their prefix; preserve + # order of first occurrence for stable group iteration ('stable' + # in MATLAB's unique). + series_names: list[str] = [] + prefixes: list[str] = [] + unique_prefixes: list[str] = [] + prefix_to_indices: dict[str, list[int]] = {} + for name in names: + m = series_re.match(name) + if not m: + continue + p = m.group(1) + idx = len(series_names) + series_names.append(name) + prefixes.append(p) + if p not in prefix_to_indices: + prefix_to_indices[p] = [] + unique_prefixes.append(p) + prefix_to_indices[p].append(idx) + + if not series_names: + return groups + + for p in unique_prefixes: + sorted_series = sorted(series_names[i] for i in prefix_to_indices[p]) + epoch = [os.path.join(directory, sorted_series[0])] + + ok = True + for ancillary in patterns[1:]: + rx = "^" + matlab_to_python_regex(ancillary).replace("#", re.escape(p)) + "$" + try: + anc_re = re.compile(rx) + except re.error: + ok = False + break + matched = sorted(n for n in names if anc_re.match(n)) + if not matched: + ok = False + break + epoch.append(os.path.join(directory, matched[0])) + + if ok: + groups.append(epoch) + + return groups + + def __repr__(self) -> str: + n_patterns = len(self._fileparameters.get("filematch", [])) + return f"ndi_file_navigator_rhd_series(patterns={n_patterns})" diff --git a/src/ndi/file/navigator/rhd_series_epochdir.py b/src/ndi/file/navigator/rhd_series_epochdir.py new file mode 100644 index 0000000..1d5c041 --- /dev/null +++ b/src/ndi/file/navigator/rhd_series_epochdir.py @@ -0,0 +1,138 @@ +""" +ndi.file.navigator.rhd_series_epochdir - Epochdir navigator for prefix-grouped .rhd recordings. + +The epochdir-organized counterpart of ndi.file.navigator.rhd_series. Each +first-level subdirectory of the session is treated as a candidate epoch +container; within each subdirectory, .rhd files that share a common prefix +but differ in a trailing variable portion (typically a YYYYMMDDHHMMSS.msec +timestamp) are grouped, and only the lexicographically earliest member of +each group is returned (the Intan reader recovers the rest of the series +from that file). Ancillary files matched through the standard '#' +substitution syntax are searched for in the same subdirectory. + +MATLAB equivalent: +ndi/+file/+navigator/rhd_series_epochdir.m +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from .epochdir import ndi_file_navigator_epochdir +from .rhd_series import ndi_file_navigator_rhd_series + + +class ndi_file_navigator_rhd_series_epochdir(ndi_file_navigator_epochdir): + """ + Epochdir navigator for prefix-grouped .rhd recordings. + + FILEPARAMETERS + The filematch list is interpreted exactly as in + ndi.file.navigator.rhd_series; the only difference is that + matching is performed independently in each first-level + subdirectory of the session rather than in the session root. + + patterns[0] - Series pattern. Contains exactly one '#' capturing + the per-epoch prefix; the remainder is a regular expression + matching the variable part of the filename. + + patterns[1:] - Ancillary patterns. '#' is replaced by the literal + (regex-escaped) prefix of the current epoch and the result + is matched as a regular expression against filenames in the + same subdirectory. The lexicographically earliest match per + pattern is appended to the epoch's file list. If any + ancillary pattern produces no match the epoch is skipped. + + The epoch identifier returned by epochid is the name of the + subdirectory that contains the epoch's files, matching the + convention of ndi.file.navigator.epochdir. + + Example: + >>> nav = ndi_file_navigator_rhd_series_epochdir( + ... session, + ... [r'#_\\d{14}\\.\\d+\\.rhd\\>', r'#\\.epochprobemap\\.ndi\\>'], + ... ) + """ + + NDI_FILENAVIGATOR_CLASS = "ndi.file.navigator.rhd_series_epochdir" + + def epochid( + self, + epoch_number: int, + epochfiles: list[str] | None = None, + ) -> str: + """ + Return the epoch identifier (subdirectory name). + + MATLAB equivalent: ndi.file.navigator.rhd_series_epochdir/epochid + + Returns the name of the subdirectory that contains the epoch's + files. If the epoch's files are ingested, the inherited + ingested-file identifier is returned instead. + + Args: + epoch_number: epoch number (1-indexed) + epochfiles: optional file list (fetched if not provided) + + Returns: + Epoch identifier string (the subdirectory name). + """ + if epochfiles is None: + epochfiles = self.getepochfiles_number(epoch_number) + + if self.isingested(epochfiles): + return self.ingestedfiles_epochid(epochfiles) + + return Path(epochfiles[0]).parent.name + + def selectfilegroups_disk(self) -> list[list[str]]: + """ + Return groups of files that comprise epochs. + + MATLAB equivalent: + ndi.file.navigator.rhd_series_epochdir/selectfilegroups_disk + + Walks the first-level subdirectories of the session and applies + the rhd_series matching rules to each. Every prefix group found + in a subdirectory contributes one epoch whose file list is the + first .rhd of the group followed by any ancillary matches found + in the same subdirectory. + + Returns: + List of epoch file groups. + """ + try: + base_path = self.path() + except ValueError: + return [] + + if not os.path.isdir(base_path): + return [] + + patterns = ndi_file_navigator_rhd_series._normalize_patterns( + self._fileparameters.get("filematch", []) + ) + if not patterns: + return [] + + epochfiles_disk: list[list[str]] = [] + try: + entries = sorted(os.listdir(base_path)) + except (FileNotFoundError, NotADirectoryError, PermissionError): + return [] + + for name in entries: + if name.startswith("."): + continue + epoch_path = os.path.join(base_path, name) + if not os.path.isdir(epoch_path): + continue + groups = ndi_file_navigator_rhd_series._group_directory(epoch_path, patterns) + for g in groups: + epochfiles_disk.append(g) + + return epochfiles_disk + + def __repr__(self) -> str: + n_patterns = len(self._fileparameters.get("filematch", [])) + return f"ndi_file_navigator_rhd_series_epochdir(patterns={n_patterns})" diff --git a/src/ndi/file/ndi_matlab_python_bridge.yaml b/src/ndi/file/ndi_matlab_python_bridge.yaml index ebe4c7f..979597b 100644 --- a/src/ndi/file/ndi_matlab_python_bridge.yaml +++ b/src/ndi/file/ndi_matlab_python_bridge.yaml @@ -428,3 +428,186 @@ classes: generate IDs from directory names. Python ad-hoc port was missing this override. Added to match MATLAB. Synchronized 2026-03-13. + + # ========================================================================= + # ndi.file.navigator.rhd_series (subclass of navigator) + # ========================================================================= + - name: navigator_rhd_series + type: class + matlab_path: "+ndi/+file/+navigator/rhd_series.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/file/navigator/rhd_series.py" + python_class: "ndi_file_navigator_rhd_series" + inherits: "ndi.file.navigator" + + properties: [] + + methods: + - name: rhd_series + kind: constructor + matlab_last_sync_hash: "2566fe4d" + input_arguments: + - name: varargin + type_matlab: "varargin" + type_python: "*args, **kwargs" + output_arguments: + - name: obj + type_python: "ndi_file_navigator_rhd_series" + decision_log: > + Exact match. Passes all arguments to parent ndi.file.navigator + constructor. Synchronized 2026-05-11. + + - name: epochid + matlab_last_sync_hash: "2566fe4d" + input_arguments: + - name: epoch_number + type_matlab: "integer" + type_python: "int" + - name: epochfiles + type_matlab: "cell array of char" + type_python: "list[str] | None" + default: "None" + output_arguments: + - name: id + type_python: "str" + decision_log: > + Exact match. Returns the prefix captured by the '#' in the + series pattern (patterns[0]). Falls back to the file stem + when no match. Honors ingested-file short-circuit. + Synchronized 2026-05-11. + + - name: selectfilegroups_disk + matlab_last_sync_hash: "2566fe4d" + input_arguments: [] + output_arguments: + - name: epochfiles_disk + type_python: "list[list[str]]" + decision_log: > + Exact match. Groups .rhd files in the session directory by + the prefix captured from patterns[0], keeps the + lexicographically earliest member of each group, and appends + one lexicographically earliest ancillary match per pattern + in patterns[1:] (with '#' replaced by the regex-escaped + prefix). Dotfile basenames are skipped. Synchronized + 2026-05-11. + + - name: normalizePatterns + kind: static + matlab_last_sync_hash: "2566fe4d" + python_name: "_normalize_patterns" + input_arguments: + - name: filematch + type_matlab: "char | cell" + type_python: "str | list[str]" + output_arguments: + - name: patterns + type_python: "list[str]" + decision_log: > + Static helper. Coerces a single string into a one-element + list. Mapped to a private Python static method since it is + an implementation detail. Synchronized 2026-05-11. + + - name: extractPrefix + kind: static + matlab_last_sync_hash: "2566fe4d" + python_name: "_extract_prefix" + input_arguments: + - name: filename + type_matlab: "char" + type_python: "str" + - name: seriesPattern + type_matlab: "char" + type_python: "str" + output_arguments: + - name: prefix + type_python: "str" + decision_log: > + Static helper. Replaces '#' with a non-greedy capture group, + anchors the regex, and returns the captured prefix or '' on + no match. Mapped to a private Python static method since it + is an implementation detail. Synchronized 2026-05-11. + + - name: groupDirectory + kind: static + matlab_last_sync_hash: "2566fe4d" + python_name: "_group_directory" + input_arguments: + - name: directory + type_matlab: "char" + type_python: "str" + - name: patterns + type_matlab: "cell array of char" + type_python: "list[str]" + output_arguments: + - name: groups + type_python: "list[list[str]]" + decision_log: > + Static helper. Applies the rhd_series matching rules to a + single directory: groups series matches by prefix (stable + first-seen order, mirroring MATLAB's unique(..., 'stable')), + keeps the lexicographically earliest series file per group, + and appends one lexicographically earliest ancillary match + per pattern in patterns[1:]. Skips dotfile basenames. Mapped + to a private Python static method since it is an + implementation detail. Synchronized 2026-05-11. + + # ========================================================================= + # ndi.file.navigator.rhd_series_epochdir (subclass of navigator.epochdir) + # ========================================================================= + - name: navigator_rhd_series_epochdir + type: class + matlab_path: "+ndi/+file/+navigator/rhd_series_epochdir.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/file/navigator/rhd_series_epochdir.py" + python_class: "ndi_file_navigator_rhd_series_epochdir" + inherits: "ndi.file.navigator.epochdir" + + properties: [] + + methods: + - name: rhd_series_epochdir + kind: constructor + matlab_last_sync_hash: "2566fe4d" + input_arguments: + - name: varargin + type_matlab: "varargin" + type_python: "*args, **kwargs" + output_arguments: + - name: obj + type_python: "ndi_file_navigator_rhd_series_epochdir" + decision_log: > + Exact match. Passes all arguments to parent + ndi.file.navigator.epochdir constructor. Synchronized + 2026-05-11. + + - name: epochid + matlab_last_sync_hash: "2566fe4d" + input_arguments: + - name: epoch_number + type_matlab: "integer" + type_python: "int" + - name: epochfiles + type_matlab: "cell array of char" + type_python: "list[str] | None" + default: "None" + output_arguments: + - name: id + type_python: "str" + decision_log: > + Exact match. Returns the name of the subdirectory that + contains the epoch's first file (matching + ndi.file.navigator.epochdir semantics). Honors + ingested-file short-circuit. Synchronized 2026-05-11. + + - name: selectfilegroups_disk + matlab_last_sync_hash: "2566fe4d" + input_arguments: [] + output_arguments: + - name: epochfiles_disk + type_python: "list[list[str]]" + decision_log: > + Exact match. Walks first-level subdirectories of the session + (skipping dotfile basenames) and applies the rhd_series + grouping rules to each, concatenating all groups. Reuses + ndi_file_navigator_rhd_series._group_directory for parity + with the flat-session navigator. Synchronized 2026-05-11. diff --git a/src/ndi/ndi_common/daq_systems/rayolab/rayo_intanSeries.json b/src/ndi/ndi_common/daq_systems/rayolab/rayo_intanSeries.json new file mode 100644 index 0000000..1b573f7 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/rayolab/rayo_intanSeries.json @@ -0,0 +1,16 @@ +{ + "Name": "rayo_intanSeries", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.daq.reader.mfdaq.ndr", + "DaqReaderFileParameters": "intan", + "MetadataReaderClass": [], + "EpochProbeMapClass": "ndi.epoch.epochprobemap_daqsystem", + "FileParameters": [ + "#_\\d{6}_\\d{6}\\.rhd\\>", + "#_\\d{6}_\\d{6}\\._epochprobemap\\.txt\\>" + ], + "MetadataReaderFileParameters": [], + "EpochProbeMapFileParameters": "#_\\d{6}_\\d{6}\\._epochprobemap\\.txt\\>", + "HasEpochDirectories": false, + "FileNavigatorClass": "ndi.file.navigator.rhd_series" +} diff --git a/src/ndi/ndi_common/daq_systems/rayolab/rayo_stim.json b/src/ndi/ndi_common/daq_systems/rayolab/rayo_stim.json new file mode 100644 index 0000000..062b144 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/rayolab/rayo_stim.json @@ -0,0 +1,16 @@ +{ + "Name": "rayo_stim", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.setup.daq.reader.mfdaq.stimulus.rayolab_intanseries", + "DaqReaderFileParameters": "intan", + "MetadataReaderClass": "ndi.daq.metadatareader.RayoLabStims", + "EpochProbeMapClass": "ndi.epoch.epochprobemap_daqsystem", + "FileParameters": [ + "#_\\d{6}_\\d{6}\\.rhd\\>", + "#_\\d{6}_\\d{6}\\._epochprobemap\\.txt\\>" + ], + "MetadataReaderFileParameters": "#_\\d{6}_\\d{6}\\._epochprobemap\\.txt\\>", + "EpochProbeMapFileParameters": "#_\\d{6}_\\d{6}\\._epochprobemap\\.txt\\>", + "HasEpochDirectories": false, + "FileNavigatorClass": "ndi.file.navigator.rhd_series" +} diff --git a/src/ndi/ndi_matlab_python_bridge.yaml b/src/ndi/ndi_matlab_python_bridge.yaml index bd58701..2e0ef55 100644 --- a/src/ndi/ndi_matlab_python_bridge.yaml +++ b/src/ndi/ndi_matlab_python_bridge.yaml @@ -12,6 +12,16 @@ project_metadata: naming_policy: "Strict MATLAB Mirror" indexing_policy: "Semantic Parity (1-based for user concepts, 0-based for internal data)" +conventions: + regex_dialect: > + All user-facing regex patterns in NDI (e.g. ndi.file.navigator + filematch, ndi.file.navigator.rhd_series series/ancillary patterns) + are written in MATLAB regex syntax. The Python port translates them + to Python re syntax internally via + ndi.util.matlab_regex.matlab_to_python_regex at the point of + compile. New python code that consumes user-provided regex must + route through this converter. + # ========================================================================= # Standalone functions # ========================================================================= @@ -34,6 +44,57 @@ functions: MATLAB uses git describe; Python does the same with subprocess fallback to __version__. Synchronized 2026-03-13. + - name: setup.lab + type: function + matlab_path: "+ndi/+setup/lab.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/setup/lab.py" + input_arguments: + - name: labName + type_matlab: "char" + type_python: "str" + - name: ref + type_matlab: "char" + type_python: "N/A (Python takes session directly)" + - name: dirname + type_matlab: "char" + type_python: "N/A (Python takes session directly)" + output_arguments: + - name: S + type_matlab: "ndi.session.dir" + type_python: "None (mutates session in-place)" + decision_log: > + Reads DAQ system JSON configs from ndi_common/daq_systems// + and adds corresponding documents to the session. MATLAB creates + the session.dir from (ref, dirname); Python takes a session that + the caller has already created and adds documents to it. + Python signature: ndi.setup.lab(session, lab_name). + + - name: setup.rayolab + type: function + matlab_path: "+ndi/+setup/rayolab.m" + matlab_last_sync_hash: "2566fe4d" + python_path: "ndi/setup/rayolab.py" + input_arguments: + - name: ref + type_matlab: "char" + type_python: "N/A (Python takes session directly)" + - name: dirname + type_matlab: "char" + type_python: "N/A (Python takes session directly)" + output_arguments: + - name: S + type_matlab: "ndi.session.dir" + type_python: "None (mutates session in-place)" + decision_log: > + Thin wrapper that calls setup.lab(session, "rayolab"). Loads the + rayo_intanSeries and rayo_stim DAQ system configs from + ndi_common/daq_systems/rayolab/. The rayo_stim system pairs with + the RayoLabStims metadata reader. MATLAB rayolab.m calls + ndi.setup.lab('rayolab', ref, dirname); Python rayolab(session) + calls lab(session, "rayolab"). Python signature: + ndi.setup.rayolab(session). + # ========================================================================= # Classes # ========================================================================= @@ -934,6 +995,21 @@ classes: none(), from_search() for MATLAB parity. Synchronized 2026-03-13. + # ========================================================================= + # ndi.preferences - Singleton user-preferences store + # ========================================================================= + - name: preferences + type: class + matlab_path: "+ndi/preferences.m" + matlab_last_sync_hash: "9061865d" + python_path: "ndi/preferences.py" + python_class: "Preferences" + inherits: "" + + decision_log: > + matlab and python both persist to ~/.ndi/NDI_Preferences.json on + all platforms (synchronized 2026-05-11). + # ========================================================================= # Not ported / Not applicable # ========================================================================= @@ -981,7 +1057,7 @@ not_applicable: scipy.optimize curve fitting directly; no wrapper needed. # ========================================================================= - # ndi.setup namespace — MATLAB-only lab configurations + # ndi.setup namespace — MATLAB lab configurations # ========================================================================= - name: setup matlab_path: "+ndi/+setup/" @@ -990,8 +1066,9 @@ not_applicable: decision_log: > Python equivalent of MATLAB lab setup wizards. The ndi.setup.lab() function reads DAQ system JSON configs from ndi_common/daq_systems/ - and creates the corresponding documents in a session. Usage: - ndi.setup.lab(session, "vhlab"). + and creates the corresponding documents in a session. Per-lab + wrappers (e.g. ndi.setup.rayolab) call lab() with a fixed name. + Usage: ndi.setup.lab(session, "vhlab"); ndi.setup.rayolab(session). # ========================================================================= # ndi.docs namespace — MATLAB documentation generation diff --git a/src/ndi/preferences.py b/src/ndi/preferences.py new file mode 100644 index 0000000..c043094 --- /dev/null +++ b/src/ndi/preferences.py @@ -0,0 +1,521 @@ +""" +ndi.preferences - Singleton store of NDI-python user preferences. + +This module mirrors the MATLAB ``ndi.preferences`` singleton (see +``src/ndi/+ndi/preferences.m`` in NDI-matlab). It manages a process-wide +collection of user-editable preferences that persist to disk as JSON. + +Each preference is represented by a :class:`PreferenceItem` with fields +matching the MATLAB struct: ``category``, ``subcategory``, ``name``, +``value``, ``default_value``, ``description``, ``type``. + +Persistence +----------- +Values are stored as JSON at:: + + ~/.ndi/NDI_Preferences.json + +The file is read once on first access and rewritten by +:func:`ndi_preferences.set` and :func:`ndi_preferences.reset`. A missing +or corrupt file is tolerated: defaults are used and a warning is issued. + +MATLAB deviates here by using ``fullfile(prefdir, 'NDI_Preferences.json')`` +(MATLAB's per-installation prefdir). Python has no equivalent of +``prefdir``, so we use ``~/.ndi/NDI_Preferences.json``. The directory +is created on first save. + +Access +------ +Most callers should use the module-level convenience functions:: + + import ndi + + value = ndi.preferences.get('Cloud.Upload.Max_File_Batch_Size') + ndi.preferences.set('Cloud.Upload.Max_File_Batch_Size', 1_000_000_000) + ndi.preferences.reset('Cloud.Upload.Max_File_Batch_Size') + ndi.preferences.reset() # reset every preference + items = ndi.preferences.list_items() # list of PreferenceItem + has = ndi.preferences.has('Cloud.Upload.Foo') + path = ndi.preferences.filename() + +The underlying singleton object can be obtained with +:func:`get_singleton` (mirrors MATLAB's ``ndi.preferences.getSingleton``). + +Conventions +----------- +Preference paths are dotted strings of the form ``Category.Name`` or +``Category.Subcategory.Name``. Lookups are case-sensitive. + +Adding a new preference +----------------------- +Edit :meth:`ndi_preferences._register_defaults` and add an +``self._add_item(category, subcategory, name, default, type_, description)`` +call. The new preference becomes available on next interpreter start (or +after calling :func:`_reset_singleton` from tests). +""" + +from __future__ import annotations + +import json +import os +import warnings +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +__all__ = [ + "PreferenceItem", + "ndi_preferences", + "get_singleton", + "preferences", + "get", + "set", + "reset", + "list_items", + "has", + "filename", +] + + +@dataclass +class PreferenceItem: + """A single preference entry. + + Mirrors the MATLAB Items struct fields. + + Attributes: + category: Top-level grouping (e.g. ``'Cloud'``). + subcategory: Optional second-level grouping (e.g. ``'Upload'``). + Empty string when unused. + name: Leaf identifier (e.g. ``'Max_File_Batch_Size'``). + value: Current value. + default_value: Value used on first run and after reset. + description: Human-readable explanation; used as tooltip by the + preferences editor and shown by :func:`list_items`. + type: Expected Python type name used to coerce values when + reloading from JSON. One of ``'float'``, ``'int'``, + ``'bool'``, ``'str'``, or ``'any'``. + """ + + category: str + subcategory: str + name: str + value: Any + default_value: Any + description: str + type: str = "any" + + def key(self) -> str: + """Build the JSON field name for this item. + + Returns ``'Category__Name'`` when subcategory is empty, + otherwise ``'Category__Subcategory__Name'``. Matches the + MATLAB ``itemKey`` static helper so files written by either + language round-trip cleanly. + """ + if not self.subcategory: + return f"{self.category}__{self.name}" + return f"{self.category}__{self.subcategory}__{self.name}" + + def path(self) -> str: + """Return the dotted public path for this preference.""" + if not self.subcategory: + return f"{self.category}.{self.name}" + return f"{self.category}.{self.subcategory}.{self.name}" + + +def _default_filename() -> Path: + """Return the absolute path of the JSON file used for persistence. + + Deviates from MATLAB ``fullfile(prefdir, 'NDI_Preferences.json')`` + because Python has no equivalent of MATLAB's per-installation + ``prefdir``. We use ``~/.ndi/NDI_Preferences.json`` and create the + directory lazily on first save. + """ + return Path.home() / ".ndi" / "NDI_Preferences.json" + + +class ndi_preferences: + """Singleton store of NDI-python user preferences. + + The class follows the singleton pattern: the first reference (via + :func:`get_singleton`) creates the instance and loads any persisted + values; every subsequent reference returns the same in-memory + object. + + Direct construction is allowed but is intended only for tests; most + code should call the module-level :func:`get`, :func:`set`, etc. + """ + + # Set of recognised type-coercion strings. + _COERCERS = {"float", "int", "bool", "str", "any"} + + def __init__(self, filename: str | os.PathLike | None = None) -> None: + """Initialise the preferences store. + + Args: + filename: Optional path of the JSON file to use. Defaults to + ``~/.ndi/NDI_Preferences.json``. Tests may pass a + temporary path here. + """ + self._filename: Path = Path(filename) if filename else _default_filename() + self._items: list[PreferenceItem] = [] + self._register_defaults() + self._load_from_disk() + + # ------------------------------------------------------------------ + # Default registration + # ------------------------------------------------------------------ + def _register_defaults(self) -> None: + """Populate :attr:`_items` with the built-in NDI preferences. + + This is the canonical place to add new preferences. Each call to + :meth:`_add_item` registers one item with its category, + subcategory, name, default value, expected type, and a short + description used by :func:`list_items` and any future + preferences editor. + """ + self._add_item( + "Cloud", + "Download", + "Max_Document_Batch_Count", + 10000, + "int", + "Maximum number of documents downloaded per batch.", + ) + self._add_item( + "Cloud", + "Upload", + "Max_Document_Batch_Count", + 100000, + "int", + "Maximum number of documents uploaded per batch.", + ) + self._add_item( + "Cloud", + "Upload", + "Max_File_Batch_Size", + 500_000_000, + "float", + "Maximum size of file batch upload in bytes (default 500 MB).", + ) + + def _add_item( + self, + category: str, + subcategory: str, + name: str, + default_value: Any, + type_: str, + description: str, + ) -> None: + """Append one preference item to :attr:`_items`. + + Both ``value`` and ``default_value`` are initialised to + ``default_value``. + """ + self._items.append( + PreferenceItem( + category=str(category), + subcategory=str(subcategory), + name=str(name), + value=default_value, + default_value=default_value, + description=str(description), + type=str(type_), + ) + ) + + # ------------------------------------------------------------------ + # Persistence + # ------------------------------------------------------------------ + def _load_from_disk(self) -> None: + """Overlay persisted values onto registered defaults. + + Called once during construction. Reads the JSON file at + :attr:`_filename`, decodes it, and copies each matching value + back into the items list (after type coercion). Items not + present in the file keep their default value. + + A missing file is silently ignored (first-run case). Any other + error is reported via :func:`warnings.warn` and defaults remain + in effect. + """ + if not self._filename.is_file(): + return + try: + text = self._filename.read_text(encoding="utf-8") + if not text.strip(): + return + data = json.loads(text) + if not isinstance(data, dict): + return + for item in self._items: + key = item.key() + if key in data: + item.value = self._coerce_type(data[key], item.type) + except Exception as exc: # noqa: BLE001 - mirror MATLAB behaviour + warnings.warn( + f"NDI:preferences:loadFailed: could not load preferences " + f"from {self._filename}: {exc}", + RuntimeWarning, + stacklevel=2, + ) + + def _save_to_disk(self) -> None: + """Write the current values to the JSON file. + + Builds a flat dict keyed by :meth:`PreferenceItem.key` and writes + it pretty-printed. Failures are reported via + :func:`warnings.warn`; the in-memory state is unaffected. + """ + payload = {item.key(): item.value for item in self._items} + try: + self._filename.parent.mkdir(parents=True, exist_ok=True) + self._filename.write_text( + json.dumps(payload, indent=2, sort_keys=True), + encoding="utf-8", + ) + except Exception as exc: # noqa: BLE001 - mirror MATLAB behaviour + warnings.warn( + f"NDI:preferences:saveFailed: could not save preferences " + f"to {self._filename}: {exc}", + RuntimeWarning, + stacklevel=2, + ) + + # ------------------------------------------------------------------ + # Lookup helpers + # ------------------------------------------------------------------ + def _find_item(self, category: str, subcategory: str, name: str) -> int: + """Locate a preference by category / subcategory / name. + + Returns: + Index into :attr:`_items` of the matching item. + + Raises: + KeyError: If no item matches. The message shows the dotted + path that was requested. Mirrors the MATLAB + ``NDI:preferences:unknownPreference`` error. + """ + for idx, item in enumerate(self._items): + if item.category == category and item.subcategory == subcategory and item.name == name: + return idx + if not subcategory: + path_str = f"{category}.{name}" + else: + path_str = f"{category}.{subcategory}.{name}" + raise KeyError(f'Unknown preference "{path_str}".') + + @staticmethod + def _parse_path(path_str: str) -> tuple[str, str, str]: + """Split a dotted preference path into its components. + + Returns: + Tuple ``(category, subcategory, name)``. ``subcategory`` is + the empty string for two-part paths. + + Raises: + ValueError: If the path is not two- or three-part. Mirrors + the MATLAB ``NDI:preferences:invalidPath`` error. + """ + parts = str(path_str).split(".") + if len(parts) == 2: + return parts[0], "", parts[1] + if len(parts) == 3: + return parts[0], parts[1], parts[2] + raise ValueError( + f"Preference path must be Category.Name or " + f'Category.Subcategory.Name (got "{path_str}").' + ) + + @classmethod + def _coerce_type(cls, raw_value: Any, type_name: str) -> Any: + """Convert a JSON-decoded value back to its declared type. + + Any conversion failure returns ``raw_value`` unchanged so a + corrupt JSON payload does not break the session. + """ + if not type_name or type_name == "any": + return raw_value + try: + if type_name == "bool": + return bool(raw_value) + if type_name == "int": + return int(raw_value) + if type_name == "float": + return float(raw_value) + if type_name == "str": + return str(raw_value) + except (TypeError, ValueError): + return raw_value + return raw_value + + # ------------------------------------------------------------------ + # Public API (instance) + # ------------------------------------------------------------------ + @property + def filename(self) -> str: + """Absolute path of the on-disk preferences file.""" + return str(self._filename) + + @property + def items(self) -> list[PreferenceItem]: + """Snapshot of all registered preference items. + + Returns a shallow copy of the internal list, so callers may not + mutate the live store through this property. + """ + return list(self._items) + + def get(self, path_str: str) -> Any: + """Return the current value of the preference at *path_str*.""" + category, subcategory, name = self._parse_path(path_str) + idx = self._find_item(category, subcategory, name) + return self._items[idx].value + + def set(self, path_str: str, value: Any) -> None: + """Update a preference and persist the change. + + No type validation is performed: the value is stored verbatim. + The ``type`` field on the item is metadata used only when + reloading the file. + """ + category, subcategory, name = self._parse_path(path_str) + idx = self._find_item(category, subcategory, name) + self._items[idx].value = value + self._save_to_disk() + + def reset(self, path_str: str | None = None) -> None: + """Restore preference defaults and persist. + + With no argument, restores every preference to its registered + ``default_value``. With *path_str*, restores only that one. + """ + if path_str is None: + for item in self._items: + item.value = item.default_value + else: + category, subcategory, name = self._parse_path(path_str) + idx = self._find_item(category, subcategory, name) + self._items[idx].value = self._items[idx].default_value + self._save_to_disk() + + def has(self, path_str: str) -> bool: + """Return ``True`` if *path_str* identifies a registered item. + + Unlike :meth:`get`, never raises: malformed paths simply return + ``False``. + """ + try: + category, subcategory, name = self._parse_path(path_str) + except ValueError: + return False + for item in self._items: + if item.category == category and item.subcategory == subcategory and item.name == name: + return True + return False + + def list_items(self) -> list[dict[str, Any]]: + """Return a list of dicts describing every registered item. + + Each dict has keys ``category``, ``subcategory``, ``name``, + ``value``, ``default_value``, ``description``, ``type``. Modifying + the returned list does not affect the singleton. + """ + return [asdict(item) for item in self._items] + + def __repr__(self) -> str: # pragma: no cover - cosmetic + groups: dict[str, dict[str, Any]] = {} + for item in self._items: + label = f"{item.subcategory}_{item.name}" if item.subcategory else item.name + groups.setdefault(item.category, {})[label] = item.value + lines = [f"ndi_preferences(filename={self._filename!s})"] + for cat, entries in groups.items(): + lines.append(f" [{cat}]") + for label, value in entries.items(): + lines.append(f" {label} = {value!r}") + return "\n".join(lines) + + +# ---------------------------------------------------------------------- +# Module-level singleton accessor (mirrors MATLAB static methods) +# ---------------------------------------------------------------------- +_singleton: ndi_preferences | None = None + + +def get_singleton() -> ndi_preferences: + """Return the shared :class:`ndi_preferences` instance. + + The first call constructs the object (which loads the JSON file + from disk); later calls reuse it. Mirrors MATLAB's + ``ndi.preferences.getSingleton``. + """ + global _singleton + if _singleton is None: + _singleton = ndi_preferences() + return _singleton + + +def preferences() -> ndi_preferences: + """Alias for :func:`get_singleton`. Pythonic accessor.""" + return get_singleton() + + +def _reset_singleton() -> None: + """Discard the cached singleton (test helper, not public API).""" + global _singleton + _singleton = None + + +# ---------------------------------------------------------------------- +# Module-level convenience functions (mirror MATLAB static methods) +# ---------------------------------------------------------------------- +def get(path_str: str) -> Any: # noqa: A001 - mirror MATLAB API + """Return the value of the preference at *path_str*. + + Equivalent to ``ndi.preferences.get(path_str)`` in MATLAB. + """ + return get_singleton().get(path_str) + + +def set(path_str: str, value: Any) -> None: # noqa: A001 - mirror MATLAB API + """Set the preference at *path_str* to *value* and persist. + + Equivalent to ``ndi.preferences.set(path_str, value)`` in MATLAB. + """ + get_singleton().set(path_str, value) + + +def reset(path_str: str | None = None) -> None: + """Reset one preference to its default, or all if *path_str* is None. + + Equivalent to ``ndi.preferences.reset`` in MATLAB. + """ + get_singleton().reset(path_str) + + +def list_items() -> list[dict[str, Any]]: + """Return a list of dicts describing every registered preference. + + Equivalent to ``ndi.preferences.list()`` in MATLAB. Renamed from + ``list`` to avoid shadowing the Python built-in. + """ + return get_singleton().list_items() + + +def has(path_str: str) -> bool: + """Return ``True`` if *path_str* identifies a registered preference. + + Equivalent to ``ndi.preferences.has(path_str)`` in MATLAB. Never + raises: malformed paths return ``False``. + """ + return get_singleton().has(path_str) + + +def filename() -> str: + """Return the absolute path of the on-disk preferences file. + + Equivalent to ``ndi.preferences.filename()`` in MATLAB. + """ + return get_singleton().filename diff --git a/src/ndi/setup/__init__.py b/src/ndi/setup/__init__.py index cbb382b..4b66f2f 100644 --- a/src/ndi/setup/__init__.py +++ b/src/ndi/setup/__init__.py @@ -1,4 +1,4 @@ -"""ndi.setup — Lab configuration and session setup utilities. +"""ndi.setup - Lab configuration and session setup utilities. Python equivalent of MATLAB's ``+ndi/+setup/`` package. @@ -6,8 +6,10 @@ import ndi ndi.setup.lab(session, "vhlab") + ndi.setup.rayolab(session) """ from .lab import lab +from .rayolab import rayolab -__all__ = ["lab"] +__all__ = ["lab", "rayolab"] diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/rayolab_intanseries.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/rayolab_intanseries.py new file mode 100644 index 0000000..064ba56 --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/rayolab_intanseries.py @@ -0,0 +1,34 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.rayolab_intanseries -- RayoLab stimulator Intan reader. + +Extends the Intan RHD reader for the RayoLab visual stimulator. The +stimulus identifier (always 1) is reported on the mk2 marker channel +of the Intan stream; the actual per-stimulus parameter set is provided +by the companion metadata reader +:class:`~ndi.daq.metadatareader.rayolab_stims.ndi_daq_metadatareader_RayoLabStims`. + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/rayolab_intanseries.m`` +""" + +from __future__ import annotations + +import logging + +from ndi.daq.reader.mfdaq.intan import ndi_daq_reader_mfdaq_intan + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_rayolab_intanseries(ndi_daq_reader_mfdaq_intan): + """RayoLab stimulator reader built on Intan RHD. + + Inherits all channel reading and sample-rate logic from the Intan + reader. The stimulus-specific behaviour (extracting stimid from the + mk2 marker channel) is handled by the metadata reader + :class:`~ndi.daq.metadatareader.rayolab_stims.ndi_daq_metadatareader_RayoLabStims` + configured alongside this reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.rayolab_intanseries" + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_rayolab_intanseries(id={self.id[:8]}...)" diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py index ee7ef73..1862f21 100644 --- a/src/ndi/setup/lab.py +++ b/src/ndi/setup/lab.py @@ -109,14 +109,25 @@ def lab(session, lab_name: str) -> None: ) session.database_add(fn_doc) - # Create daqreader document — use the specialised document type - # for readers that require extra properties (e.g. ndr needs - # daqreader_ndr.ndr_reader_string so MATLAB can reconstruct it). + # Create daqreader document. NDR-family readers (the base + # ndi.daq.reader.mfdaq.ndr and any subclass thereof, e.g. + # ndi.setup.daq.reader.mfdaq.stimulus.rayolab_intanseries) need + # the daqreader_ndr doc shape with ndr_reader_string so MATLAB + # can reconstruct them; the matlab base ndr constructor reads + # document_properties.daqreader_ndr.ndr_reader_string and errors + # out on the generic daqreader shape. + # + # Signal: an NDR-family entry in the lab JSON sets + # DaqReaderFileParameters (the ndr "reader string" -- typically + # the format tag like "intan"). Use that as the trigger so we + # catch subclasses without hard-coding their class names. reader_file_params = config.get("DaqReaderFileParameters", "") if isinstance(reader_file_params, list): reader_file_params = reader_file_params[0] if reader_file_params else "" - if reader_class == "ndi.daq.reader.mfdaq.ndr": + is_ndr_family = reader_class == "ndi.daq.reader.mfdaq.ndr" or bool(reader_file_params) + + if is_ndr_family: dr_doc = session.newdocument( "daq/daqreader_ndr", **{ diff --git a/src/ndi/setup/rayolab.py b/src/ndi/setup/rayolab.py new file mode 100644 index 0000000..7302245 --- /dev/null +++ b/src/ndi/setup/rayolab.py @@ -0,0 +1,38 @@ +"""ndi.setup.rayolab - Initialize a session with RayoLab DAQ systems. + +Python equivalent of MATLAB's ``ndi.setup.rayolab()``. Thin wrapper around +:func:`ndi.setup.lab` that loads the RayoLab DAQ system JSON configs from +``ndi_common/daq_systems/rayolab/``. + +The RayoLab setup currently includes two DAQ systems: + +- ``rayo_intanSeries`` -- raw Intan RHD acquisition, grouped by filename + prefix via ``ndi.file.navigator.rhd_series``, read with + ``ndi.daq.reader.mfdaq.ndr`` (the NDR ``intan`` reader). +- ``rayo_stim`` -- the RayoLab stimulator, paired with the + :class:`~ndi.daq.metadatareader.ndi_daq_metadatareader_RayoLabStims` + metadata reader (which always returns ``stimid = 1``). + +MATLAB equivalent: ``src/ndi/+ndi/+setup/rayolab.m`` + +Example:: + + import ndi + session = ndi.session.dir("exp001", "/path/to/session") + ndi.setup.rayolab(session) +""" + +from __future__ import annotations + +from .lab import lab + + +def rayolab(session) -> None: + """Add the RayoLab DAQ systems to an NDI session. + + Parameters + ---------- + session : ndi.session.session_base + The NDI session to add DAQ systems to. + """ + lab(session, "rayolab") diff --git a/src/ndi/util/__init__.py b/src/ndi/util/__init__.py index 3d3b07c..dce6fd8 100644 --- a/src/ndi/util/__init__.py +++ b/src/ndi/util/__init__.py @@ -23,6 +23,7 @@ from .hexDiff import hexDiff from .hexDiffBytes import hexDiffBytes from .hexDump import hexDump +from .matlab_regex import matlab_to_python_regex from .rehydrateJSONNanNull import rehydrateJSONNanNull from .session_summary import sessionSummary from .unwrapTableCellContent import unwrapTableCellContent @@ -39,6 +40,7 @@ "hexDiff", "hexDiffBytes", "hexDump", + "matlab_to_python_regex", "rehydrateJSONNanNull", "sessionSummary", "unwrapTableCellContent", diff --git a/src/ndi/util/matlab_regex.py b/src/ndi/util/matlab_regex.py new file mode 100644 index 0000000..d9bc330 --- /dev/null +++ b/src/ndi/util/matlab_regex.py @@ -0,0 +1,89 @@ +""" +ndi.util.matlab_regex - Translate MATLAB regex syntax to Python ``re`` syntax. + +All user-facing NDI regex (file navigator ``filematch`` patterns, +ndi.file.navigator.rhd_series series/ancillary patterns, etc.) is +written in MATLAB regex syntax. The cross-language bridge yaml +documents MATLAB regex as the contract for those fields. This module +performs a minimum-viable translation from a MATLAB pattern to an +equivalent Python ``re`` pattern. + +Translations performed +---------------------- +* ``\\<`` -> ``\\b`` (MATLAB start-of-word anchor) +* ``\\>`` -> ``\\b`` (MATLAB end-of-word anchor) +* ``(?...)`` -> ``(?P...)`` (named capturing group) + +The boundary translations only fire when the backslash is the regex +escape; a doubled backslash (``\\\\>``) which encodes a literal +backslash followed by ``>`` is left alone, as is a bare ``<`` or ``>`` +that is not preceded by a backslash. ``(?<=...)`` lookbehinds are +preserved because they start with ``(?<=`` / ``(?``. + +The function is idempotent: a pattern that has already been through +the translator is unchanged when passed through a second time, because +``\\b`` has no MATLAB-only construct and ``(?P...)`` does not +match the MATLAB named-group regex. + +MATLAB regex features NOT yet handled (extend this module to add them) +--------------------------------------------------------------------- +* POSIX character classes inside a class, e.g. ``[[:alpha:]]``, + ``[[:digit:]]``, ``[[:space:]]``. Python ``re`` has no direct + equivalent; would need expansion to explicit class contents. +* ``\\k`` backreference to a named group (MATLAB) vs. + ``(?P=name)`` in Python. +* ``(?#comment)`` is shared between the two flavors but with subtle + rules; not normalized here. +* MATLAB's ``\\oNNN`` octal escape (Python uses ``\\NNN``). +* MATLAB's ``(?@command)``, ``(?(condition)yes|no)`` conditional + patterns, and ``(?>...)`` atomic groups when used with MATLAB-only + options. +* MATLAB ``(?i)``/``(?-i)`` inline flag scoping differs slightly from + Python; we do not rewrite inline flags. +* MATLAB ``\\cX`` control-character escape (Python uses ``\\xNN``). +""" + +from __future__ import annotations + +import re + +__all__ = ["matlab_to_python_regex"] + + +# Match an odd number of backslashes followed by '<' or '>'. The negative +# lookbehind anchors the backslash run to a non-backslash boundary so the +# count is exact: an even-length prefix is each pair of literal backslashes +# and the final \\ is the escape that owns the < or >. +_WORD_BOUNDARY_RE = re.compile(r"(?(?:\\\\)*)\\(?P[<>])") + +# MATLAB named group (?...) -> Python (?P...). Exclude +# (?<=...) and (?[A-Za-z_][A-Za-z0-9_]*)>") + + +def matlab_to_python_regex(pattern: str) -> str: + """Translate a MATLAB regex pattern to an equivalent Python ``re`` pattern. + + The translation is minimum-viable: it covers MATLAB start/end-of-word + boundaries (``\\<`` / ``\\>``) and MATLAB named capturing groups + (``(?...)``). All other constructs pass through unchanged. + + The function is pure and idempotent. + + Args: + pattern: MATLAB-flavored regex string. + + Returns: + Python ``re``-compatible regex string. + """ + if not isinstance(pattern, str): + raise TypeError(f"matlab_to_python_regex expected str, got {type(pattern).__name__}") + + def _boundary_sub(m: re.Match[str]) -> str: + return f"{m.group('bs')}\\b" + + result = _WORD_BOUNDARY_RE.sub(_boundary_sub, pattern) + result = _NAMED_GROUP_RE.sub(r"(?P<\g>", result) + return result diff --git a/tests/_matlab_license_guard.py b/tests/_matlab_license_guard.py new file mode 100644 index 0000000..c7a2fd2 --- /dev/null +++ b/tests/_matlab_license_guard.py @@ -0,0 +1,84 @@ +""" +Shared MATLAB BYOL test guard. + +This module exists for ONE reason: to make it impossible for a misconfigured +CI run to silently destroy a real MATLAB BYOL license registration on the +test cloud account. + +The MATLAB BYOL endpoints (allocate / set / clear) mutate user state that +lives outside the test sandbox -- a stale ENI or orphaned license costs real +money and time to recover. The MATLAB equivalent test +(tests/+ndi/+unittest/+cloud/MatlabLicenseTest.m) uses fatalAssert in +TestClassSetup to refuse to run when NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE +is unset. This helper mirrors that guard for pytest. + +Semantics of NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE: + - empty / unset -> fail (configuration error; refuse to run at all) + - "true" / "1" -> the test account already has a license that MUST be + preserved; destructive tests skip themselves and the + teardown does NOT call clearMatlabLicense + - "false" / "0" -> the test account has no license; destructive tests + run end-to-end and the teardown cleans up + +This module is shared between test_cloud_matlab_license.py and +test_cloud_hello_matlab.py so the same env-var contract is enforced +identically wherever the destructive endpoints might be touched. +""" + +from __future__ import annotations + +import os + +ENV_VAR = "NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE" + +_TRUE_VALUES = {"true", "1"} +_FALSE_VALUES = {"false", "0"} + +_FATAL_MESSAGE = ( + "LOCAL CONFIGURATION ERROR: NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE is not set.\n" + "This variable MUST be set explicitly for any MATLAB BYOL test to run, " + "because these tests call DELETE /users/me/matlab-license and would " + "otherwise silently destroy a real registered license file on a " + "misconfigured CI account.\n" + 'Set it to "true" if the test account already has a MATLAB license ' + "(destructive tests will be skipped) or " + '"false" if it does not (destructive tests will run and clean up).' +) + + +def _raw_value() -> str: + return os.environ.get(ENV_VAR, "") + + +def fatal_check_license_env() -> None: + """Raise RuntimeError if NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE is unset. + + Call at module import time so the failure surfaces as a collection + error, mirroring MATLAB's fatalAssertNotEmpty in TestClassSetup. + + We intentionally raise RuntimeError instead of calling pytest.skip: + an unset env var is a misconfiguration, not a 'no credentials + available' no-op. Pytest treats an exception during collection as + an ERROR rather than a SKIP, which is exactly what we want -- a + green CI run that destroyed someone's license would be the + nightmare scenario this guard prevents. + """ + if not _raw_value().strip(): + raise RuntimeError(_FATAL_MESSAGE) + + +def user_has_existing_license() -> bool: + """Return True iff env var explicitly says the test account has a license. + + Accepted true values: ``"true"``, ``"1"`` (case-insensitive). + Anything else (including the false values) returns False. Matches + the MATLAB check ``strcmpi(flag,"true") || flag=="1"``. + """ + val = _raw_value().strip().lower() + return val in _TRUE_VALUES + + +def env_value_is_recognized() -> bool: + """True iff env var is set to one of the four recognized values.""" + val = _raw_value().strip().lower() + return val in _TRUE_VALUES or val in _FALSE_VALUES diff --git a/tests/symmetry/make_artifacts/file/__init__.py b/tests/symmetry/make_artifacts/file/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/make_artifacts/file/test_rhd_series_navigator.py b/tests/symmetry/make_artifacts/file/test_rhd_series_navigator.py new file mode 100644 index 0000000..9c40909 --- /dev/null +++ b/tests/symmetry/make_artifacts/file/test_rhd_series_navigator.py @@ -0,0 +1,111 @@ +"""Generate symmetry artifacts for ndi.file.navigator.rhd_series. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+file/rhdSeriesNavigator.m + +Builds two prefix groups of fake .rhd files (A_* and B_*) plus matching +_epochprobemap sidecars, runs ndi.file.navigator.rhd_series against +them, and writes:: + + rhd_series_navigator.json - navigator class, fileparameters, and + per-epoch {epochid, files} entries. + fixture/ - copy of the on-disk .rhd / probemap + files so the cross-language read side + can re-walk them with a fresh navigator. + +Artifact root: + /NDI/symmetryTest/pythonArtifacts/file/rhdSeriesNavigator/ + testRhdSeriesNavigator/ +""" + +import json +import os +import shutil + +import pytest + +from ndi.file.navigator.rhd_series import ndi_file_navigator_rhd_series +from ndi.session.dir import ndi_session_dir +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "file" / "rhdSeriesNavigator" / "testRhdSeriesNavigator" + +# These patterns are identical to the matlab makeArtifacts pair so a +# matlab-written rhd_series_navigator.json round-trips cleanly. The first +# pattern groups .rhd files by their prefix (capture); the second matches +# the _epochprobemap.txt sidecar for the same prefix. +FILEPARAMETERS = [ + r"#_\d{8}_\d{6}\.rhd\>", + r"#_\d{8}_\d{6}\._epochprobemap\.txt\>", +] + +# Fake epoch fixture: two prefix groups (A, B), two .rhd files per group, +# one epochprobemap per group (matches the matlab fixture byte-for-byte). +FIXTURE_FILES = [ + "A_20260101_120000.rhd", + "A_20260101_120500.rhd", + "A_20260101_120000._epochprobemap.txt", + "B_20260101_130000.rhd", + "B_20260101_130500.rhd", + "B_20260101_130000._epochprobemap.txt", +] + + +class TestRhdSeriesNavigatorMakeArtifacts: + """Mirror of ndi.symmetry.makeArtifacts.file.rhdSeriesNavigator.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a session dir populated with the fake .rhd fixture.""" + session_dir = tmp_path / "rhd_series_session" + session_dir.mkdir() + for name in FIXTURE_FILES: + (session_dir / name).touch() + self.session_path = session_dir + self.session = ndi_session_dir("exp1", session_dir) + + def test_rhd_series_navigator(self): + artifact_dir = ARTIFACT_DIR + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + artifact_dir.mkdir(parents=True, exist_ok=True) + + # Construct the navigator with the series + ancillary patterns. + # MATLAB lookup: nav = ndi.file.navigator.rhd_series(session, fileparameters); + nav = ndi_file_navigator_rhd_series(self.session, FILEPARAMETERS) + + groups = nav.selectfilegroups_disk() + assert ( + len(groups) == 2 + ), f"Expected 2 epoch groups from rhd_series navigator, got {len(groups)}." + + epoch_info = [] + for i, files in enumerate(groups): + eid = nav.epochid(i + 1, files) + rel = [os.path.basename(f) for f in files] + epoch_info.append({"epochid": eid, "files": rel}) + + # Copy the on-disk fixture into the artifact dir so the matlab + # readArtifacts side (and the python read side) can re-walk it. + fixture_dir = artifact_dir / "fixture" + fixture_dir.mkdir(exist_ok=True) + for name in os.listdir(self.session_path): + if name.startswith("."): + continue + src = self.session_path / name + if src.is_file(): + shutil.copy2(str(src), str(fixture_dir / name)) + + nav_doc = { + "navigator_class": "ndi.file.navigator.rhd_series", + "fileparameters": FILEPARAMETERS, + "epochs": epoch_info, + } + nav_path = artifact_dir / "rhd_series_navigator.json" + nav_path.write_text( + json.dumps(nav_doc, indent=2, allow_nan=True), + encoding="utf-8", + ) + + assert nav_path.is_file() + assert fixture_dir.is_dir() diff --git a/tests/symmetry/make_artifacts/session/test_blank_session_rayolab.py b/tests/symmetry/make_artifacts/session/test_blank_session_rayolab.py new file mode 100644 index 0000000..e6343a1 --- /dev/null +++ b/tests/symmetry/make_artifacts/session/test_blank_session_rayolab.py @@ -0,0 +1,86 @@ +"""Generate symmetry artifacts for a blank rayolab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+session/blankSessionRayolab.m + +Builds a blank NDI session configured with the rayolab DAQ systems +(rayo_intanSeries + rayo_stim, both using the rhd_series file navigator), +then copies the entire session tree plus a sessionSummary.json manifest +and per-document JSON dumps to the shared symmetry artifact directory at:: + + /NDI/symmetryTest/pythonArtifacts/session/blankSessionRayolab/ + testBlankSessionRayolab/ + +Artifacts are left on disk so the MATLAB readArtifacts suite and the +python read_artifacts suite can verify cross-language parity. +""" + +import json +import shutil + +import pytest + +import ndi.setup +from ndi.query import ndi_query +from ndi.session.dir import ndi_session_dir +from ndi.util import sessionSummary +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "session" / "blankSessionRayolab" / "testBlankSessionRayolab" + + +class TestBlankSessionRayolab: + """Mirror of ndi.symmetry.makeArtifacts.session.blankSessionRayolab.""" + + @pytest.fixture(autouse=True) + def _setup(self, tmp_path): + """Create a blank session with rayolab DAQ systems.""" + session_dir = tmp_path / "exp1" + session_dir.mkdir() + + session = ndi_session_dir("exp1", session_dir) + session.cache.clear() + + # ndi.setup.rayolab(session) mutates the session in place, + # adding the rayo_intanSeries and rayo_stim DAQ systems. + ndi.setup.rayolab(session) + + self.session = session + + def test_blank_session_rayolab(self): + """Export the blank rayolab session to the symmetry artifact dir.""" + artifact_dir = ARTIFACT_DIR + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + # Re-open the session to capture any auto-generated documents. + session_path = self.session.path + self.session = ndi_session_dir("exp1", session_path) + + summary = sessionSummary(self.session) + summary_json = json.dumps(summary, indent=2, allow_nan=True) + + # Copy the entire session tree into the artifact dir, matching the + # matlab side's copyfile(SessionPath, artifactDir). + shutil.copytree(str(session_path), str(artifact_dir)) + + # Per-document JSON dumps so the matlab readArtifacts side can + # cross-check individual documents if desired. + json_docs_dir = artifact_dir / "jsonDocuments" + json_docs_dir.mkdir(exist_ok=True) + + docs = self.session.database_search(ndi_query("base.id").match("(.*)")) + for doc in docs: + props = doc.document_properties + doc_path = json_docs_dir / f"{doc.id}.json" + doc_path.write_text( + json.dumps(props, indent=2, allow_nan=True), + encoding="utf-8", + ) + + summary_path = artifact_dir / "sessionSummary.json" + summary_path.write_text(summary_json, encoding="utf-8") + + assert artifact_dir.exists() + assert summary_path.exists() + assert json_docs_dir.exists() diff --git a/tests/symmetry/make_artifacts/util/__init__.py b/tests/symmetry/make_artifacts/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/make_artifacts/util/test_preferences.py b/tests/symmetry/make_artifacts/util/test_preferences.py new file mode 100644 index 0000000..3f3b6e2 --- /dev/null +++ b/tests/symmetry/make_artifacts/util/test_preferences.py @@ -0,0 +1,98 @@ +"""Generate symmetry artifacts for ndi.preferences. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+util/preferences.m + +Writes a flat JSON payload of all registered preferences (defaults plus +two overrides) into the symmetry artifact directory so the MATLAB and +Python readArtifacts suites can verify cross-language parity of the +ndi.preferences on-disk format. + +Artifact root: + /NDI/symmetryTest/pythonArtifacts/util/preferences/testPreferences/ + +Files written: + NDI_Preferences.json - pretty-printed flat dict keyed by + Category__[Subcategory__]Name. + preferences_overrides.json - just the override entries, used by the + readArtifacts test to verify + round-trip values. +""" + +import json +import shutil + +import pytest + +import ndi.preferences as ndi_preferences +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "util" / "preferences" / "testPreferences" + +# Two non-default values used by the readArtifacts pair to assert that +# overrides round-trip cleanly through JSON. Keys use the +# Category__Subcategory__Name encoding that matches ndi.preferences on disk. +OVERRIDES = { + "Cloud__Upload__Max_File_Batch_Size": 123456789, + "Cloud__Download__Max_Document_Batch_Count": 42, +} + + +class TestPreferencesMakeArtifacts: + """Mirror of ndi.symmetry.makeArtifacts.util.preferences.""" + + def test_preferences(self): + """Write NDI_Preferences.json + preferences_overrides.json to disk.""" + artifact_dir = ARTIFACT_DIR + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + artifact_dir.mkdir(parents=True, exist_ok=True) + + # Load the live preferences singleton so we use the real registered + # defaults; build a flat JSON payload using the same + # 'Category__Subcategory__Name' encoding that ndi.preferences uses on + # disk. We do NOT use ndi.preferences.set() here because that would + # mutate the user's real prefdir copy; instead we write directly to + # the artifact dir. + items = ndi_preferences.list_items() + assert items, "ndi.preferences.list_items() returned no entries." + + payload = {} + for item in items: + subcategory = item.get("subcategory", "") or "" + if subcategory: + key = f"{item['category']}__{subcategory}__{item['name']}" + else: + key = f"{item['category']}__{item['name']}" + if key in OVERRIDES: + payload[key] = OVERRIDES[key] + else: + payload[key] = item["default_value"] + + # Sanity-check that every override key actually exists in the live + # preferences singleton. If an override does not correspond to a + # registered preference, the matlab readArtifacts side would still + # see the value (because make writes it), but cross-language + # symmetry would silently drift. xfail in that case rather than + # quietly papering over the gap. + missing = [k for k in OVERRIDES if k not in payload] + if missing: + pytest.xfail( + "ndi.preferences is missing entries the matlab side registers: " + + ", ".join(missing) + ) + + prefs_path = artifact_dir / "NDI_Preferences.json" + prefs_path.write_text( + json.dumps(payload, indent=2), + encoding="utf-8", + ) + + overrides_path = artifact_dir / "preferences_overrides.json" + overrides_path.write_text( + json.dumps(OVERRIDES, indent=2), + encoding="utf-8", + ) + + assert prefs_path.is_file() + assert overrides_path.is_file() diff --git a/tests/symmetry/make_artifacts/util/test_profile.py b/tests/symmetry/make_artifacts/util/test_profile.py new file mode 100644 index 0000000..80a1cc6 --- /dev/null +++ b/tests/symmetry/make_artifacts/util/test_profile.py @@ -0,0 +1,74 @@ +"""Generate symmetry artifacts for ndi.cloud.profile. + +Python equivalent of: + tests/+ndi/+symmetry/+makeArtifacts/+util/profile.m + +Uses the in-memory backend so no AES file or OS keyring secret is ever +persisted. Writes a NDI_Cloud_Profiles.json file containing exactly one +profile with:: + + Nickname = 'SymmetryTest' + Email = 'test@example.org' + Stage = 'test' (forced; ndi.cloud.profile.set_stage only + accepts 'prod'/'dev', so we mutate the entry + in memory to satisfy the contract) + +PasswordSecret is stripped before persisting; no secret leaves the process. +Artifact root: + /NDI/symmetryTest/pythonArtifacts/util/profile/testProfile/ +""" + +import json +import shutil +from dataclasses import asdict + +import ndi.cloud.profile as ndi_profile +from tests.symmetry.conftest import PYTHON_ARTIFACTS + +ARTIFACT_DIR = PYTHON_ARTIFACTS / "util" / "profile" / "testProfile" + + +class TestProfileMakeArtifacts: + """Mirror of ndi.symmetry.makeArtifacts.util.profile.""" + + def test_profile(self): + artifact_dir = ARTIFACT_DIR + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + artifact_dir.mkdir(parents=True, exist_ok=True) + + # Use the in-memory backend so no AES file or keyring secret is + # persisted. Reset the singleton state so we start clean. + ndi_profile.use_backend("memory") + ndi_profile.reset() + + uid = ndi_profile.add("SymmetryTest", "test@example.org", "not-a-real-secret") + ndi_profile.set_stage(uid, "dev") + ndi_profile.set_stage(uid, "prod") + + # Per spec, Stage='test' is required even though set_stage only + # accepts 'prod'/'dev'. Mutate the in-memory profile directly to + # satisfy the symmetry contract. + entry = ndi_profile.get(uid) + entry_dict = asdict(entry) + entry_dict["Stage"] = "test" + # Do NOT persist PasswordSecret to disk; the readArtifacts side + # asserts this field is absent. + entry_dict.pop("PasswordSecret", None) + + payload = { + "Profiles": entry_dict, + "DefaultUID": "", + } + + out_file = artifact_dir / "NDI_Cloud_Profiles.json" + out_file.write_text( + json.dumps(payload, indent=2), + encoding="utf-8", + ) + + # Clean up the in-memory singleton state so other tests are not + # affected. No secret leaves the process. + ndi_profile.reset() + + assert out_file.is_file() diff --git a/tests/symmetry/read_artifacts/file/__init__.py b/tests/symmetry/read_artifacts/file/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/read_artifacts/file/test_rhd_series_navigator.py b/tests/symmetry/read_artifacts/file/test_rhd_series_navigator.py new file mode 100644 index 0000000..3ce9631 --- /dev/null +++ b/tests/symmetry/read_artifacts/file/test_rhd_series_navigator.py @@ -0,0 +1,112 @@ +"""Read and verify symmetry artifacts for ndi.file.navigator.rhd_series. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+file/rhdSeriesNavigator.m + +Loads rhd_series_navigator.json plus the fixture/ subdirectory written +by the make pair (in either pythonArtifacts or matlabArtifacts), re-walks +the fixture with a fresh ndi.file.navigator.rhd_series, and asserts that: + +- the navigator_class field is ndi.file.navigator.rhd_series; +- the same number of epoch groups is produced; +- the same set of epoch IDs is produced; +- the lexicographically-first file in each group matches. +""" + +import json +import os +import shutil + +import pytest + +from ndi.file.navigator.rhd_series import ndi_file_navigator_rhd_series +from ndi.session.dir import ndi_session_dir +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + + +class TestRhdSeriesNavigatorReadArtifacts: + """Mirror of ndi.symmetry.readArtifacts.file.rhdSeriesNavigator.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE / source_type / "file" / "rhdSeriesNavigator" / "testRhdSeriesNavigator" + ) + + def test_rhd_series_navigator(self, tmp_path, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + + nav_json = artifact_dir / "rhd_series_navigator.json" + assert ( + nav_json.is_file() + ), f"rhd_series_navigator.json missing in {source_type} artifact dir." + + expected = json.loads(nav_json.read_text(encoding="utf-8")) + assert expected.get("navigator_class") == "ndi.file.navigator.rhd_series", ( + f"Navigator class mismatch in {source_type}: " + f"got {expected.get('navigator_class')!r}." + ) + + fixture_dir = artifact_dir / "fixture" + assert fixture_dir.is_dir(), f"fixture directory missing in {source_type}." + + # Copy the fixture into a clean per-test session dir and re-walk + # with a fresh navigator. This mirrors the matlab pair which + # creates tempdir()/NDI/test_rhdSeriesNavigator_read for the same + # purpose. + session_path = tmp_path / "rhd_series_session_read" + session_path.mkdir() + for name in os.listdir(fixture_dir): + if name.startswith("."): + continue + src = fixture_dir / name + if src.is_file(): + shutil.copy2(str(src), str(session_path / name)) + session = ndi_session_dir("exp1", session_path) + + # The matlab side may have JSON-encoded a single-element cell as a + # bare string instead of a list; coerce to list either way. + fileparameters = expected.get("fileparameters", []) + if isinstance(fileparameters, str): + fileparameters = [fileparameters] + else: + fileparameters = list(fileparameters) + + nav = ndi_file_navigator_rhd_series(session, fileparameters) + groups = nav.selectfilegroups_disk() + + expected_epochs = expected.get("epochs", []) + assert len(groups) == len(expected_epochs), ( + f"Number of epoch groups mismatch in {source_type}: " + f"actual={len(groups)} expected={len(expected_epochs)}." + ) + + actual_ids = sorted(nav.epochid(i + 1, files) for i, files in enumerate(groups)) + expected_ids = sorted(str(e["epochid"]) for e in expected_epochs) + assert actual_ids == expected_ids, ( + f"Epoch ids mismatch in {source_type}: " + f"actual={actual_ids!r} expected={expected_ids!r}." + ) + + actual_firsts = sorted(os.path.basename(files[0]) for files in groups) + expected_firsts = [] + for e in expected_epochs: + files = e.get("files", []) + if isinstance(files, str): + expected_firsts.append(files) + elif files: + expected_firsts.append(str(files[0])) + expected_firsts.sort() + assert actual_firsts == expected_firsts, ( + f"Group first-file mismatch in {source_type}: " + f"actual={actual_firsts!r} expected={expected_firsts!r}." + ) diff --git a/tests/symmetry/read_artifacts/session/test_blank_session_rayolab.py b/tests/symmetry/read_artifacts/session/test_blank_session_rayolab.py new file mode 100644 index 0000000..5101f9a --- /dev/null +++ b/tests/symmetry/read_artifacts/session/test_blank_session_rayolab.py @@ -0,0 +1,120 @@ +"""Read and verify symmetry artifacts for a blank rayolab NDI session. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+session/blankSessionRayolab.m + +Loads the artifact directory written by the rayolab makeArtifacts pair +(in either pythonArtifacts or matlabArtifacts) and verifies: + +- the session opens cleanly via ndi.session.dir; +- exactly two DAQ systems are present, named 'rayo_intanSeries' and + 'rayo_stim'; +- both DAQ systems use ndi.file.navigator.rhd_series as their file + navigator (asserted via the MATLAB-compatible NDI_FILENAVIGATOR_CLASS + string for cross-language parity); +- the sessionSummary.json manifest, if present, matches the live + summary produced by sessionSummary() (with daq-system order + normalized to avoid spurious failures). +""" + +import json + +import pytest + +from ndi.session.dir import ndi_session_dir +from ndi.util import compareSessionSummary, sessionSummary +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE +from tests.symmetry.read_artifacts.session._summary_helpers import ( + sort_daq_systems_by_name, +) + +EXPECTED_DAQ_NAMES = {"rayo_intanSeries", "rayo_stim"} +RHD_SERIES_CLASS = "ndi.file.navigator.rhd_series" + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + + +class TestBlankSessionRayolab: + """Mirror of ndi.symmetry.readArtifacts.session.blankSessionRayolab.""" + + def _artifact_dir(self, source_type: str): + return ( + SYMMETRY_BASE + / source_type + / "session" + / "blankSessionRayolab" + / "testBlankSessionRayolab" + ) + + def _open_session(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + return artifact_dir, ndi_session_dir("exp1", artifact_dir) + + def test_blank_session_rayolab_daq_systems(self, source_type): + """Verify rayo_intanSeries + rayo_stim are present with rhd_series.""" + _artifact_dir, session = self._open_session(source_type) + + daqs = session.daqsystem_load(name="(.*)") + if daqs is None: + daqs = [] + elif not isinstance(daqs, list): + daqs = [daqs] + + assert len(daqs) == 2, ( + f"Expected 2 DAQ systems in {source_type} rayolab session, " f"got {len(daqs)}." + ) + + names = {getattr(d, "name", "") for d in daqs} + for expected in EXPECTED_DAQ_NAMES: + assert expected in names, ( + f"Expected DAQ system {expected!r} not found in {source_type}; " + f"got {sorted(names)!r}." + ) + + # Each DAQ system should use ndi.file.navigator.rhd_series. We + # compare against the MATLAB-compatible class string carried by + # the python navigator (NDI_FILENAVIGATOR_CLASS) so a matlab- or + # python-written session both pass this check. + for d in daqs: + nav = getattr(d, "filenavigator", None) + assert nav is not None, f"DAQ system {d.name!r} has no filenavigator in {source_type}." + nav_class = getattr(nav, "NDI_FILENAVIGATOR_CLASS", "") + assert nav_class == RHD_SERIES_CLASS, ( + f"DAQ system {d.name!r} should use {RHD_SERIES_CLASS} in " + f"{source_type}, got {nav_class!r}." + ) + + def test_blank_session_rayolab_summary(self, source_type): + """Verify the on-disk session summary matches the live summary.""" + artifact_dir, session = self._open_session(source_type) + + summary_path = artifact_dir / "sessionSummary.json" + if not summary_path.is_file(): + pytest.skip( + f"sessionSummary.json not found in {source_type} artifact dir; " + f"skipping summary comparison." + ) + + expected_summary = json.loads(summary_path.read_text(encoding="utf-8")) + actual_summary = sessionSummary(session) + + sort_daq_systems_by_name(actual_summary) + sort_daq_systems_by_name(expected_summary) + + report = compareSessionSummary( + actual_summary, + expected_summary, + excludeFiles=["sessionSummary.json", "jsonDocuments"], + ) + assert len(report) == 0, ( + f"Session summary mismatch against {source_type} generated " + f"artifacts:\n" + "\n".join(report) + ) diff --git a/tests/symmetry/read_artifacts/util/__init__.py b/tests/symmetry/read_artifacts/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/symmetry/read_artifacts/util/test_preferences.py b/tests/symmetry/read_artifacts/util/test_preferences.py new file mode 100644 index 0000000..7d6eda1 --- /dev/null +++ b/tests/symmetry/read_artifacts/util/test_preferences.py @@ -0,0 +1,92 @@ +"""Read and verify symmetry artifacts for ndi.preferences. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+util/preferences.m + +Loads NDI_Preferences.json and preferences_overrides.json from either the +pythonArtifacts root (own pair) or the matlabArtifacts root (cross-language) +and asserts: + +- every key in the JSON file matches the Category__[Subcategory__]Name + encoding; +- every override key from preferences_overrides.json is present in + NDI_Preferences.json with the same value; +- every registered ndi.preferences default key has a corresponding entry + in the JSON payload (sanity check that the make side wrote a complete + snapshot of the singleton). +""" + +import json + +import pytest + +import ndi.preferences as ndi_preferences +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + """Parameterize over matlabArtifacts / pythonArtifacts.""" + return request.param + + +class TestPreferencesReadArtifacts: + """Mirror of ndi.symmetry.readArtifacts.util.preferences.""" + + def _artifact_dir(self, source_type: str): + return SYMMETRY_BASE / source_type / "util" / "preferences" / "testPreferences" + + def test_preferences(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + + prefs_path = artifact_dir / "NDI_Preferences.json" + assert prefs_path.is_file(), f"NDI_Preferences.json missing in {source_type}." + + payload = json.loads(prefs_path.read_text(encoding="utf-8")) + assert isinstance( + payload, dict + ), f"NDI_Preferences.json in {source_type} did not decode to a dict." + assert payload, f"No preference keys in {source_type}." + + # Every key must use the Category__[Subcategory__]Name encoding + # (2 or 3 components when split on '__'). + for key in payload: + parts = key.split("__") + assert len(parts) in (2, 3), ( + f"Preference key {key!r} does not match " + f"Category__[Subcategory__]Name in {source_type}." + ) + + # Load override metadata and verify round-trip values + overrides_path = artifact_dir / "preferences_overrides.json" + if overrides_path.is_file(): + overrides = json.loads(overrides_path.read_text(encoding="utf-8")) + assert isinstance(overrides, dict) + for okey, oval in overrides.items(): + assert okey in payload, f"Override key {okey} missing from prefs in {source_type}." + assert payload[okey] == oval, ( + f"Override value mismatch for {okey} in {source_type}: " + f"expected {oval!r}, got {payload[okey]!r}." + ) + + # The live ndi.preferences defaults should all appear in the file. + # This is the symmetry check: if the matlab side writes a key the + # python side does not register (or vice versa), we want a clear + # signal. Missing-on-python keys are tolerated by recording them + # via pytest's verbose output, since the make-side may have been + # written by a different language with a richer preference set. + items = ndi_preferences.list_items() + for item in items: + subcategory = item.get("subcategory", "") or "" + if subcategory: + key = f"{item['category']}__{subcategory}__{item['name']}" + else: + key = f"{item['category']}__{item['name']}" + assert key in payload, ( + f"Python-registered preference {key!r} missing from " f"{source_type} payload." + ) diff --git a/tests/symmetry/read_artifacts/util/test_profile.py b/tests/symmetry/read_artifacts/util/test_profile.py new file mode 100644 index 0000000..0c809da --- /dev/null +++ b/tests/symmetry/read_artifacts/util/test_profile.py @@ -0,0 +1,68 @@ +"""Read and verify symmetry artifacts for ndi.cloud.profile. + +Python equivalent of: + tests/+ndi/+symmetry/+readArtifacts/+util/profile.m + +Loads NDI_Cloud_Profiles.json from either pythonArtifacts (own pair) or +matlabArtifacts (cross-language) and verifies that the file describes +exactly one SymmetryTest profile with Stage='test' and no PasswordSecret. +The Profiles field is allowed to be a single dict (matlab encoding of one +struct) or a list of dicts (general case). +""" + +import json + +import pytest + +from tests.symmetry.conftest import SOURCE_TYPES, SYMMETRY_BASE + + +@pytest.fixture(params=SOURCE_TYPES) +def source_type(request): + return request.param + + +class TestProfileReadArtifacts: + """Mirror of ndi.symmetry.readArtifacts.util.profile.""" + + def _artifact_dir(self, source_type: str): + return SYMMETRY_BASE / source_type / "util" / "profile" / "testProfile" + + def test_profile(self, source_type): + artifact_dir = self._artifact_dir(source_type) + if not artifact_dir.exists(): + pytest.skip( + f"Artifact directory from {source_type} does not exist. " + f"Run the corresponding makeArtifacts suite first." + ) + + profiles_file = artifact_dir / "NDI_Cloud_Profiles.json" + assert profiles_file.is_file(), f"NDI_Cloud_Profiles.json missing in {source_type}." + + payload = json.loads(profiles_file.read_text(encoding="utf-8")) + assert isinstance( + payload, dict + ), f"Top-level payload should be a JSON object in {source_type}." + assert "Profiles" in payload, f"Profiles field missing in {source_type}." + assert "DefaultUID" in payload, f"DefaultUID field missing in {source_type}." + + profiles = payload["Profiles"] + if isinstance(profiles, dict): + arr = [profiles] + elif isinstance(profiles, list): + arr = profiles + else: + pytest.fail(f"Unexpected Profiles type {type(profiles).__name__} in " f"{source_type}.") + + assert len(arr) == 1, ( + f"Expected exactly one profile entry in {source_type}, " f"got {len(arr)}." + ) + p = arr[0] + assert isinstance(p, dict), f"Profile entry should be an object in {source_type}." + assert p.get("UID"), f"UID missing in {source_type} profile." + assert p.get("Nickname") == "SymmetryTest", f"Nickname mismatch in {source_type}." + assert p.get("Email") == "test@example.org", f"Email mismatch in {source_type}." + assert p.get("Stage") == "test", f"Stage should be 'test' in {source_type}." + assert ( + "PasswordSecret" not in p + ), f"PasswordSecret must not be persisted on disk in {source_type}." diff --git a/tests/test_cloud_hello_matlab.py b/tests/test_cloud_hello_matlab.py new file mode 100644 index 0000000..786a6ab --- /dev/null +++ b/tests/test_cloud_hello_matlab.py @@ -0,0 +1,141 @@ +""" +Live test for the hello-matlab-v1 compute pipeline. + +Ported from MATLAB tests/+ndi/+unittest/+cloud/+compute/HelloMatlabTest.m +(matlab HEAD 2566fe4d, 2026-05-11). + +WHY TWO ENV-VAR GATES: +1. NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE -- same safety guard as + test_cloud_matlab_license.py. This file imports the same fatal + check so a misconfigured CI cannot run ANY MATLAB BYOL test + without an explicit declaration of license state. With + HAS_LICENSE=true we ASSUME a license is already present and SKIP + any license-allocation steps (mirror of MATLAB semantics). With + HAS_LICENSE=false we still require a registered license out-of-band + (we do not auto-allocate one in this test), and skip gracefully if + none is found. +2. NDI_CLOUD_RUN_HELLO_MATLAB -- opt-in because the pipeline launches a + real EC2 instance (~2-4 min, billable). Matches MATLAB's + ``assumeFail(isempty(getenv('NDI_CLOUD_RUN_HELLO_MATLAB')))``. +""" + +from __future__ import annotations + +import os + +import pytest + +from tests._matlab_license_guard import ( + fatal_check_license_env, + user_has_existing_license, +) + +# Fatal license-deletion safety guard (see _matlab_license_guard for +# rationale). Even though this test does not directly call +# clearMatlabLicense, it imports the same helper to force consistent CI +# configuration across all MATLAB BYOL tests. +fatal_check_license_env() + +_has_creds = bool(os.environ.get("NDI_CLOUD_USERNAME") and os.environ.get("NDI_CLOUD_PASSWORD")) +pytestmark = pytest.mark.skipif(not _has_creds, reason="NDI cloud credentials not set") + + +USER_HAS_EXISTING_LICENSE = user_has_existing_license() + + +@pytest.fixture(scope="module") +def cloud_client(): + from ndi.cloud.auth import login + from ndi.cloud.client import CloudClient + from ndi.cloud.config import CloudConfig + + config = CloudConfig.from_env() + config = login(config=config) + assert config.is_authenticated, "Login failed -- no token received" + return CloudClient(config) + + +class TestHelloMatlab: + """Mirror of MATLAB HelloMatlabTest.""" + + def test_hello_matlab_flow(self, cloud_client): + """Run hello-matlab-v1 end-to-end and verify success. + + Opt-in via NDI_CLOUD_RUN_HELLO_MATLAB (matches MATLAB). Requires + a registered MATLAB BYOL license; we sanity-check that before + starting the pipeline so a missing license fails with a clear + message instead of a server-side MATLAB_LICENSE_REQUIRED error + 2 minutes later. + """ + if not os.environ.get("NDI_CLOUD_RUN_HELLO_MATLAB", "").strip(): + pytest.skip( + "Set NDI_CLOUD_RUN_HELLO_MATLAB=1 to run the hello-matlab-v1 " + "end-to-end test (launches a real EC2 instance and requires " + "a registered MATLAB BYOL license)." + ) + + # --- 1. Sanity-check that a license is registered. --------------- + # When HAS_LICENSE=true we ASSUME a license already exists (this + # mirrors MATLAB: with HAS_LICENSE=true the destructive + # allocate/set steps are skipped and an existing registration is + # taken as given). When HAS_LICENSE=false the user must still + # have arranged for a license to be present out-of-band before + # running this opt-in test; we verify and skip if not. + from ndi.cloud.api.users import getMatlabLicense + + license_status = getMatlabLicense(client=cloud_client) + assert isinstance( + license_status, dict + ), f"Expected dict from getMatlabLicense, got {type(license_status)}" + files = license_status.get("files") or [] + if not files: + pytest.skip( + "No MATLAB license is registered for this user. Register " + "one with ndi.cloud.api.users.allocateMatlabLicenseMac + " + "setMatlabLicense before running this test. " + f"getMatlabLicense response: {license_status}" + ) + + if USER_HAS_EXISTING_LICENSE: + # HAS_LICENSE=true: assume the existing registration is the + # one we want to use. Do NOT call allocate/set here. + pass + + # --- 2. Run the hello-matlab-v1 pipeline end-to-end. ------------- + # The MATLAB test calls ndi.cloud.helloMatlab(). The Python port + # of that orchestration helper has not landed yet; once it does + # it is expected to expose a matching name on the ndi.cloud + # package. Until then, skip cleanly so this test file is ready + # to activate without modification. + try: + from ndi.cloud import helloMatlab + except ImportError: + pytest.skip( + "ndi.cloud.helloMatlab has not yet been ported from MATLAB. " + "This test file is ready to activate as soon as the " + "orchestration helper lands." + ) + + result = helloMatlab( + timeout_seconds=1200, + poll_interval_seconds=15, + verbose=True, + client=cloud_client, + ) + + # helloMatlab return shape: MATLAB returns (success, sessionId, + # statusMessage, sessionDoc); the Python wrapper is expected to + # return a dict with the same fields. Support both forms here so + # the test does not break on minor shape differences when the + # port lands. + if isinstance(result, dict): + success = result.get("success") + status_message = result.get("statusMessage", "") + else: + success = result[0] + status_message = result[2] if len(result) > 2 else "" + + assert success, ( + "hello-matlab-v1 pipeline did not complete successfully. " + f"statusMessage={status_message!r} full result={result}" + ) diff --git a/tests/test_cloud_matlab_license.py b/tests/test_cloud_matlab_license.py new file mode 100644 index 0000000..1fad21b --- /dev/null +++ b/tests/test_cloud_matlab_license.py @@ -0,0 +1,262 @@ +""" +Live tests for the MATLAB BYOL license endpoints. + +Ported from MATLAB tests/+ndi/+unittest/+cloud/MatlabLicenseTest.m +(matlab HEAD 2566fe4d, 2026-05-11). + +WHY THE EXTRA GUARDS: +These tests exercise allocate / set / clear of a real MATLAB license +registration. clearMatlabLicense releases the AWS ENI and removes the +license file the user uploaded to MathWorks's allocator. Running this +suite against a real account that has a real license configured would +silently destroy that license. The MATLAB side enforces an opt-in via +fatalAssert in TestClassSetup; we mirror that behaviour here with the +helper in tests._matlab_license_guard. + +Environment variables consumed: + NDI_CLOUD_USERNAME / NDI_CLOUD_PASSWORD - skip if absent (standard + live-test pattern, matches test_cloud_live.py). + NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE - REQUIRED, see guard helper. + "true" -> only the read-only getMatlabLicense test runs. + "false" -> destructive allocate/clear lifecycle runs end-to-end. + unset -> module fails to import (collection error). + +WHY THE DESTRUCTIVE TESTS ARE PINNED TO ONE PYTHON VERSION: +The CI test matrix runs python 3.10, 3.11, and 3.12 in parallel against +the SAME shared cloud account. allocate_and_clear_lifecycle and +setMatlabLicense_rejects_invalid_file both POST a new ENI/MAC, then +DELETE it; running 3 of them concurrently races their own state +(observed: MAC-mismatch failures and HTTP 500s from DELETE on +already-cleared registrations). We pin the destructive tests to +python 3.12 only; the read-only getMatlabLicense test stays on every +matrix entry so we still cross-check the GET path on each interpreter. +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +from tests._matlab_license_guard import ( + fatal_check_license_env, + user_has_existing_license, +) + +# --------------------------------------------------------------------------- +# Fatal license-deletion safety guard (runs at module import). +# +# An unset NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE is a CONFIGURATION ERROR, +# not a 'skip silently' condition. We raise here so pytest reports a +# collection failure rather than a green run that destroyed someone's +# license. This mirrors MATLAB fatalAssertNotEmpty in TestClassSetup. +# --------------------------------------------------------------------------- +fatal_check_license_env() + +# --------------------------------------------------------------------------- +# Standard live-cloud skip (no creds -> module skipped). Order matters: +# the fatal check above runs first so that an unset HAS_LICENSE env var is +# reported even when credentials are also missing. +# --------------------------------------------------------------------------- +_has_creds = bool(os.environ.get("NDI_CLOUD_USERNAME") and os.environ.get("NDI_CLOUD_PASSWORD")) +pytestmark = pytest.mark.skipif(not _has_creds, reason="NDI cloud credentials not set") + + +USER_HAS_EXISTING_LICENSE = user_has_existing_license() + +# The destructive tests share a single cloud account across the python +# matrix; pin them to one interpreter to avoid the 3.10/3.11/3.12 jobs +# racing each other's allocate/clear cycles. The read-only test still +# runs on every matrix entry. +_DESTRUCTIVE_PINNED_VERSION = (3, 12) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def cloud_client(): + """Authenticated CloudClient, scoped to the module.""" + from ndi.cloud.auth import login + from ndi.cloud.client import CloudClient + from ndi.cloud.config import CloudConfig + + config = CloudConfig.from_env() + config = login(config=config) + assert config.is_authenticated, "Login failed -- no token received" + return CloudClient(config) + + +@pytest.fixture() +def destructive_teardown(cloud_client): + """Per-test finalizer that defensively calls clearMatlabLicense. + + Mirrors MATLAB TestMethodTeardown.maybeClear: if the destructive test + set the 'we allocated something' flag before failing mid-flight, the + teardown still releases the ENI on the way out. The flag stays False + by default so the teardown is a no-op when the test never allocated. + + The flag is exposed as a mutable holder so the test can flip it to + True at the moment of allocation and back to False once it has + successfully cleared the registration itself. + + NEVER runs effectively when NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE=true: + in that mode the destructive tests are skipped entirely (see + _require_destructive_safe) BEFORE the holder is ever flipped, so + clear_on_teardown stays False and the teardown is a no-op. This + guarantees we never call DELETE on an existing license we were + supposed to preserve. + """ + from ndi.cloud.api.users import clearMatlabLicense + + holder = {"clear_on_teardown": False} + yield holder + if holder["clear_on_teardown"]: + try: + clearMatlabLicense(client=cloud_client) + except Exception: + # Best-effort cleanup; the test itself will report the failure. + pass + + +def _require_destructive_safe(): + """Skip when a destructive test would race or destroy a real license.""" + if sys.version_info[:2] != _DESTRUCTIVE_PINNED_VERSION: + pytest.skip( + "Destructive BYOL tests pinned to Python " + f"{_DESTRUCTIVE_PINNED_VERSION[0]}.{_DESTRUCTIVE_PINNED_VERSION[1]} " + "to avoid parallel-matrix interference on the shared cloud account " + "(allocate/clear races between python versions produce flaky " + "MAC-mismatch and HTTP 500 failures)." + ) + if USER_HAS_EXISTING_LICENSE: + pytest.skip( + "Skipped: NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE=true. " + "Destructive allocate/clear would mutate an existing " + "registration that must be preserved." + ) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestMatlabLicense: + """Mirror of MATLAB MatlabLicenseTest.""" + + def test_getMatlabLicense(self, cloud_client): + """Read-only check that runs in BOTH modes and on every python version. + + Asserts that the server returns a sane MatlabLicenseStatus dict + and that its ``files`` array agrees with the env-var declaration: + HAS_LICENSE=true -> files must be non-empty + HAS_LICENSE=false -> files must be empty (else the env var is + lying and the destructive tests would + destroy a real license). + """ + from ndi.cloud.api.users import getMatlabLicense + + answer = getMatlabLicense(client=cloud_client) + assert isinstance(answer, dict), f"Expected dict, got {type(answer)}" + + files = answer.get("files") or [] + has_files = bool(files) + + if USER_HAS_EXISTING_LICENSE: + assert has_files, ( + "NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE=true but the user " + f"has no registered files. Response: {answer}" + ) + else: + assert not has_files, ( + "Expected an empty registration but the test user already " + "has MATLAB license files registered. Set " + "NDI_CLOUD_TEST_USER_HAS_MATLAB_LICENSE=true to preserve " + f"them. Response: {answer}" + ) + + def test_allocate_and_clear_lifecycle(self, cloud_client, destructive_teardown): + """POST -> GET -> DELETE -> GET round-trip. + + Pinned to a single python version (see module docstring); also + skipped when HAS_LICENSE=true (DELETE would destroy a real + registration). The teardown finalizer releases the ENI we + ourselves allocated if any of the intermediate asserts fail. + """ + _require_destructive_safe() + + from ndi.cloud.api.users import ( + allocateMatlabLicenseMac, + clearMatlabLicense, + getMatlabLicense, + ) + + # --- allocate ------------------------------------------------------- + alloc = allocateMatlabLicenseMac(client=cloud_client) + # From here on, a failure MUST trigger teardown cleanup so we don't + # strand an ENI in our AWS account. + destructive_teardown["clear_on_teardown"] = True + + assert isinstance(alloc, dict), f"allocate returned {type(alloc)}: {alloc}" + assert alloc.get("macAddress"), f"Allocate response did not include a macAddress: {alloc}" + + # --- get reflects the allocation ----------------------------------- + after_alloc = getMatlabLicense(client=cloud_client) + assert isinstance(after_alloc, dict) + if "mode" in after_alloc and after_alloc["mode"] is not None: + assert ( + str(after_alloc["mode"]) == "dedicated" + ), f"Expected mode=dedicated after allocate, got {after_alloc}" + if "macAddress" in after_alloc and after_alloc["macAddress"]: + assert str(after_alloc["macAddress"]) == str( + alloc["macAddress"] + ), f"MAC mismatch: allocate={alloc} get={after_alloc}" + + # --- clear (releases the ENI) -------------------------------------- + clearMatlabLicense(client=cloud_client) + # The clear just succeeded; the teardown clear is now redundant. + destructive_teardown["clear_on_teardown"] = False + + # --- get reflects empty -------------------------------------------- + after_clear = getMatlabLicense(client=cloud_client) + assert isinstance(after_clear, dict) + files_after = after_clear.get("files") or [] + assert not files_after, ( + "Files array should be empty after clearMatlabLicense. " f"Response: {after_clear}" + ) + + def test_setMatlabLicense_rejects_invalid_file(self, cloud_client, destructive_teardown): + """Negative test: PUT with a bogus lic body should return HTTP 400. + + Pinned to a single python version (see module docstring); also + skipped when HAS_LICENSE=true (even a 400-rejected PUT could + disturb an existing registration if server semantics change). + Requires a prior allocate so the server actually reaches file + validation rather than rejecting on 'no MAC allocated'. + """ + _require_destructive_safe() + + from ndi.cloud.api.users import ( + allocateMatlabLicenseMac, + setMatlabLicense, + ) + from ndi.cloud.exceptions import CloudAPIError + + # Allocate a MAC so the PUT exercises *file* validation. + alloc = allocateMatlabLicenseMac(client=cloud_client) + assert isinstance(alloc, dict), f"allocate returned {type(alloc)}: {alloc}" + destructive_teardown["clear_on_teardown"] = True + + bogus_file = "this is not a real MATLAB license file" + with pytest.raises(CloudAPIError) as excinfo: + setMatlabLicense(bogus_file, mode="dedicated", release="R2024b", client=cloud_client) + # Verify the server actually rejected the FILE (HTTP 400), not the + # request shape or auth. + assert excinfo.value.status_code == 400, ( + f"Expected HTTP 400 for invalid lic; got {excinfo.value.status_code}: " + f"{excinfo.value}" + ) diff --git a/tests/test_matlab_regex.py b/tests/test_matlab_regex.py new file mode 100644 index 0000000..a05e727 --- /dev/null +++ b/tests/test_matlab_regex.py @@ -0,0 +1,90 @@ +"""Tests for ndi.util.matlab_regex.matlab_to_python_regex.""" + +from __future__ import annotations + +import re + +import pytest + +from ndi.util.matlab_regex import matlab_to_python_regex + + +class TestMatlabToPythonRegex: + def test_end_of_word_boundary(self): + assert matlab_to_python_regex(r"foo\>") == r"foo\b" + + def test_start_of_word_boundary(self): + assert matlab_to_python_regex(r"\") == r"\bfoo\b" + + def test_navigator_pattern_from_failing_symmetry_test(self): + translated = matlab_to_python_regex(r"#_\d{8}_\d{6}\.rhd\>") + assert translated == r"#_\d{8}_\d{6}\.rhd\b" + # And it actually matches when the '#' is substituted. + compiled = re.compile("^" + translated.replace("#", "(.+?)") + "$") + m = compiled.match("A_20260101_120000.rhd") + assert m is not None + assert m.group(1) == "A" + + def test_named_group_translation(self): + assert ( + matlab_to_python_regex(r"(?\d{4})-(?\d{2})") + == r"(?P\d{4})-(?P\d{2})" + ) + + def test_named_group_actually_compiles(self): + py = matlab_to_python_regex(r"(?\d{4})") + m = re.match(py, "2026") + assert m is not None + assert m.group("year") == "2026" + + def test_idempotent(self): + once = matlab_to_python_regex(r"\(?\d+)") + twice = matlab_to_python_regex(once) + assert once == twice + + def test_passthrough_python_pattern(self): + # Already-Python patterns must be left alone. + assert matlab_to_python_regex(r"\bfoo\b") == r"\bfoo\b" + assert matlab_to_python_regex(r"(?P\d+)") == r"(?P\d+)" + + def test_bare_angle_brackets_not_touched(self): + # '<' and '>' not preceded by an odd number of backslashes + # must NOT be converted. + assert matlab_to_python_regex("a>b") == "a>b" + assert matlab_to_python_regex("a]") == "[<>]" + + def test_escaped_backslash_then_gt_left_alone(self): + # r"\\>" is a literal backslash followed by '>' — NOT a word + # boundary. Must stay as a literal-backslash + '>'. + assert matlab_to_python_regex(r"\\>") == r"\\>" + assert matlab_to_python_regex(r"\\<") == r"\\<" + + def test_odd_backslashes_before_gt_still_a_boundary(self): + # r"\\\>" -> literal backslash + word boundary. + # That is: two backslashes (literal '\') then '\>'. + # Expected output: r"\\\b" (literal '\' + '\b'). + assert matlab_to_python_regex(r"\\\>") == r"\\\b" + + def test_lookbehind_not_renamed(self): + # (?<=...) is a lookbehind, NOT a named group; must pass through. + assert matlab_to_python_regex(r"(?<=foo)bar") == r"(?<=foo)bar" + assert matlab_to_python_regex(r"(?[A-Z])_\d+\>" + out = matlab_to_python_regex(src) + assert out == r"\b(?P[A-Z])_\d+\b" + m = re.match(out, "A_123") + assert m is not None + assert m.group("prefix") == "A"