Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 40 additions & 12 deletions src/ndi/cloud/api/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,20 @@
_Client = Annotated[CloudClient | None, SkipValidation()]


def _resolve_org_id(org_id: str | None, client: CloudClient) -> str:
"""Return *org_id* if given, otherwise pull it from client config."""
if org_id:
return org_id
resolved = getattr(client, "config", None)
if resolved and getattr(resolved, "org_id", ""):
return resolved.org_id
raise ValueError(
"org_id is required but was not provided and could not be "
"resolved from the client config. Either pass org_id explicitly "
"or set NDI_CLOUD_ORGANIZATION_ID in the environment."
)


@_auto_client
@validate_call(config=VALIDATE_CONFIG)
def getDataset(dataset_id: CloudId, *, client: _Client = None) -> dict[str, Any]:
Expand All @@ -29,16 +43,22 @@ def getDataset(dataset_id: CloudId, *, client: _Client = None) -> dict[str, Any]


@_auto_client
@validate_call(config=VALIDATE_CONFIG)
def createDataset(
org_id: NonEmptyStr,
name: NonEmptyStr,
org_id: str | None = None,
name: str = "",
description: str = "",
*,
client: _Client = None,
**kwargs: Any,
) -> dict[str, Any]:
"""POST /organizations/{organizationId}/datasets"""
"""POST /organizations/{organizationId}/datasets

If *org_id* is omitted it is resolved from the client's config
(populated automatically during login), matching MATLAB behaviour.
"""
org_id = _resolve_org_id(org_id, client)
if not name:
raise ValueError("name is required")
body: dict[str, Any] = {"name": name}
if description:
body["description"] = description
Expand Down Expand Up @@ -92,15 +112,19 @@ def deleteDataset(


@_auto_client
@validate_call(config=VALIDATE_CONFIG)
def listDatasets(
org_id: NonEmptyStr,
page: PageNumber = 1,
page_size: PageSize = 1000,
org_id: str | None = None,
page: int = 1,
page_size: int = 1000,
*,
client: _Client = None,
) -> dict[str, Any]:
"""GET /organizations/{organizationId}/datasets?page=&pageSize="""
"""GET /organizations/{organizationId}/datasets?page=&pageSize=

If *org_id* is omitted it is resolved from the client's config
(populated automatically during login), matching MATLAB behaviour.
"""
org_id = _resolve_org_id(org_id, client)
return client.get(
"/organizations/{organizationId}/datasets",
params={"page": page, "pageSize": page_size},
Expand All @@ -112,9 +136,13 @@ def listDatasets(


@_auto_client
@validate_call(config=VALIDATE_CONFIG)
def listAllDatasets(org_id: NonEmptyStr, *, client: _Client = None) -> APIResponse:
"""Auto-paginate through all datasets for an organisation."""
def listAllDatasets(org_id: str | None = None, *, client: _Client = None) -> APIResponse:
"""Auto-paginate through all datasets for an organisation.

If *org_id* is omitted it is resolved from the client's config
(populated automatically during login), matching MATLAB behaviour.
"""
org_id = _resolve_org_id(org_id, client)
all_datasets: list[dict[str, Any]] = []
page = 1
while page <= _MAX_PAGES:
Expand Down
4 changes: 3 additions & 1 deletion src/ndi/cloud/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,9 @@ def downloadGenericFiles(
all_docs = []
for doc_id in ndi_document_ids:
q = Query("base.id", "exact_string", doc_id, "")
results = ndi_dataset.database_search(q) if hasattr(ndi_dataset, "database_search") else []
results = (
ndi_dataset.database_search(q) if hasattr(ndi_dataset, "database_search") else []
)
all_docs.extend(results)

if not all_docs:
Expand Down
4 changes: 3 additions & 1 deletion src/ndi/cloud/internal.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,9 @@ def duplicateDocuments(
for i in range(0, len(doc_ids_to_delete), maximum_delete_batch_size):
batch = doc_ids_to_delete[i : i + maximum_delete_batch_size]
batch_num = i // maximum_delete_batch_size + 1
total_batches = (len(doc_ids_to_delete) + maximum_delete_batch_size - 1) // maximum_delete_batch_size
total_batches = (
len(doc_ids_to_delete) + maximum_delete_batch_size - 1
) // maximum_delete_batch_size
if verbose:
print(f"Deleting batch {batch_num} of {total_batches}...")
try:
Expand Down
1 change: 0 additions & 1 deletion src/ndi/cloud/orchestration.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,6 @@ def newDataset(
# Re-export from upload module (MATLAB: ndi.cloud.upload.scanForUpload)
from .upload import scanForUpload # noqa: F401


# ---------------------------------------------------------------------------
# Private sync helpers
# ---------------------------------------------------------------------------
Expand Down
4 changes: 1 addition & 3 deletions src/ndi/cloud/sync/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,9 +477,7 @@ def twoWaySync(
failed.append(doc_id)

# 4. Download remote-only docs
docs, dl_failed = downloadNdiDocuments(
cloud_dataset_id, remote_ids, to_download, client=client
)
docs, dl_failed = downloadNdiDocuments(cloud_dataset_id, remote_ids, to_download, client=client)
saved = _save_downloaded_docs(ds_path, docs)
report["downloaded"] = saved
failed.extend(dl_failed)
Expand Down
3 changes: 2 additions & 1 deletion tests/test_cloud_live.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,8 @@ def _cleanup_stale_pytest_datasets(client, cloud_config):
result = listDatasets(cloud_config.org_id, client=client)
datasets = result.get("datasets", [])
stale = [
ds for ds in datasets
ds
for ds in datasets
if ds.get("name", "").startswith("NDI_PYTEST") and ds.get("_id", ds.get("id", ""))
]
if stale:
Expand Down
Loading