Skip to content

Commit 80d3ff7

Browse files
stevevanhooserstevevanhooser
authored
Merge branch 'main' into refactor-tests-structure-1426338253063092469
2 parents c374c0b + b1142fc commit 80d3ff7

63 files changed

Lines changed: 2371 additions & 26 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎comparison_table.md‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -135,32 +135,32 @@
135135
| ndi.cloud.internal.getActiveToken | No | No |
136136
| ndi.cloud.internal.getCloudDatasetIdForLocalDataset | No | No |
137137
| ndi.cloud.internal.getTokenExpiration | No | No |
138-
| ndi.cloud.internal.getUploadedDocumentIds | No | No |
139-
| ndi.cloud.internal.getUploadedFileIds | No | No |
140-
| ndi.cloud.internal.getWeboptionsWithAuthHeader | No | No |
141-
| ndi.cloud.sync.enum.SyncMode | No | No |
142-
| ndi.cloud.sync.internal.index.createSyncIndexStruct | No | No |
143-
| ndi.cloud.sync.internal.index.getIndexFilepath | No | No |
144-
| ndi.cloud.sync.internal.index.readSyncIndex | No | No |
145-
| ndi.cloud.sync.internal.index.updateSyncIndex | No | No |
146-
| ndi.cloud.sync.internal.index.writeSyncIndex | No | No |
147-
| ndi.cloud.sync.internal.Constants | No | No |
148-
| ndi.cloud.sync.internal.datasetSessionIdFromDocs | No | No |
149-
| ndi.cloud.sync.internal.deleteLocalDocuments | No | No |
150-
| ndi.cloud.sync.internal.deleteRemoteDocuments | No | No |
151-
| ndi.cloud.sync.internal.downloadNdiDocuments | No | No |
138+
| ndi.cloud.internal.getUploadedDocumentIds | Yes | Yes |
139+
| ndi.cloud.internal.getUploadedFileIds | Yes | Yes |
140+
| ndi.cloud.internal.getWeboptionsWithAuthHeader | Yes | Yes |
141+
| ndi.cloud.sync.enum.SyncMode | Yes | Yes |
142+
| ndi.cloud.sync.internal.index.createSyncIndexStruct | Yes | Yes |
143+
| ndi.cloud.sync.internal.index.getIndexFilepath | Yes | Yes |
144+
| ndi.cloud.sync.internal.index.readSyncIndex | Yes | Yes |
145+
| ndi.cloud.sync.internal.index.updateSyncIndex | Yes | Yes |
146+
| ndi.cloud.sync.internal.index.writeSyncIndex | Yes | Yes |
147+
| ndi.cloud.sync.internal.Constants | Yes | Yes |
148+
| ndi.cloud.sync.internal.datasetSessionIdFromDocs | Yes | Yes |
149+
| ndi.cloud.sync.internal.deleteLocalDocuments | Yes | Yes |
150+
| ndi.cloud.sync.internal.deleteRemoteDocuments | Yes | Yes |
151+
| ndi.cloud.sync.internal.downloadNdiDocuments | Yes | Yes |
152152
| ndi.cloud.sync.internal.filesNotYetUploaded | No | No |
153-
| ndi.cloud.sync.internal.getFileUidsFromDocuments | No | No |
154-
| ndi.cloud.sync.internal.listLocalDocuments | No | No |
155-
| ndi.cloud.sync.internal.listRemoteDocumentIds | No | No |
156-
| ndi.cloud.sync.internal.updateFileInfoForLocalFiles | No | No |
157-
| ndi.cloud.sync.internal.updateFileInfoForRemoteFiles | No | No |
158-
| ndi.cloud.sync.internal.uploadFilesForDatasetDocuments | No | No |
159-
| ndi.cloud.sync.SyncOptions | No | No |
160-
| ndi.cloud.sync.downloadNew | No | No |
161-
| ndi.cloud.sync.mirrorFromRemote | No | No |
162-
| ndi.cloud.sync.mirrorToRemote | No | No |
163-
| ndi.cloud.sync.twoWaySync | No | No |
153+
| ndi.cloud.sync.internal.getFileUidsFromDocuments | Yes | Yes |
154+
| ndi.cloud.sync.internal.listLocalDocuments | Yes | Yes |
155+
| ndi.cloud.sync.internal.listRemoteDocumentIds | Yes | Yes |
156+
| ndi.cloud.sync.internal.updateFileInfoForLocalFiles | Yes | Yes |
157+
| ndi.cloud.sync.internal.updateFileInfoForRemoteFiles | Yes | Yes |
158+
| ndi.cloud.sync.internal.uploadFilesForDatasetDocuments | Yes | Yes |
159+
| ndi.cloud.sync.SyncOptions | Yes | Yes |
160+
| ndi.cloud.sync.downloadNew | Yes | Yes |
161+
| ndi.cloud.sync.mirrorFromRemote | Yes | Yes |
162+
| ndi.cloud.sync.mirrorToRemote | Yes | Yes |
163+
| ndi.cloud.sync.twoWaySync | Yes | Yes |
164164
| ndi.cloud.sync.uploadNew | No | No |
165165
| ndi.cloud.sync.validate | No | No |
166166
| ndi.cloud.ui.dialog.selectCloudDataset | No | No |

‎src/ndi/cloud/api/files/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from .get_file_details import get_file_details
2+
from .get_file_upload_url import get_file_upload_url

‎src/ndi/cloud/api/files/get_file_upload_url.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from ..implementation.files.get_file_upload_url import GetFileUploadURL as GetFileUploadURLImpl
2+
3+
def get_file_upload_url(dataset_id, file_uid, organization_id=None):
4+
"""
5+
Gets the upload URL for a file in a dataset.
6+
7+
Args:
8+
dataset_id (str): The ID of the dataset.
9+
file_uid (str): The UID of the file.
10+
organization_id (str, optional): The ID of the organization.
11+
12+
Returns:
13+
tuple: (success, answer, response, url)
14+
"""
15+
api_call = GetFileUploadURLImpl(dataset_id, file_uid, organization_id)
16+
return api_call.execute()

‎src/ndi/cloud/api/implementation/files/get_file_upload_url.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
from ...call import Call
2+
from ... import url
3+
from ....authenticate import authenticate
4+
import requests
5+
import json
6+
7+
class GetFileUploadURL(Call):
8+
"""
9+
Implementation class for getting a file upload URL.
10+
"""
11+
12+
def __init__(self, dataset_id, file_uid, organization_id=None):
13+
"""
14+
Creates a new GetFileUploadURL API call object.
15+
16+
Args:
17+
dataset_id (str): The ID of the dataset.
18+
file_uid (str): The UID of the file.
19+
organization_id (str, optional): The ID of the organization. If not provided, it will be retrieved from the environment.
20+
"""
21+
self.dataset_id = dataset_id
22+
self.file_uid = file_uid
23+
self.organization_id = organization_id
24+
self.endpoint_name = 'get_file_upload_url'
25+
26+
def execute(self):
27+
"""
28+
Performs the API call.
29+
"""
30+
token = authenticate()
31+
32+
# Pass organization_id if available, otherwise url.get_url will try env var
33+
kwargs = {
34+
'dataset_id': self.dataset_id,
35+
'file_uid': self.file_uid
36+
}
37+
if self.organization_id:
38+
kwargs['organization_id'] = self.organization_id
39+
40+
try:
41+
api_url = url.get_url(self.endpoint_name, **kwargs)
42+
except ValueError as e:
43+
# Likely missing organizationId
44+
return False, str(e), None, None
45+
46+
headers = {
47+
'Accept': 'application/json',
48+
'Authorization': f'Bearer {token}'
49+
}
50+
51+
response = requests.get(api_url, headers=headers)
52+
53+
if response.status_code == 200:
54+
return True, response.json(), response, api_url
55+
else:
56+
try:
57+
answer = response.json()
58+
except json.JSONDecodeError:
59+
answer = response.text
60+
return False, answer, response, api_url

‎src/ndi/cloud/api/url.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def get_url(endpoint_name, **kwargs):
3030
'uid': kwargs.get('file_uid'),
3131
'datasetId': kwargs.get('dataset_id'),
3232
'documentId': kwargs.get('document_id'),
33-
'organizationId': kwargs.get('organization_id'),
33+
'organizationId': kwargs.get('organization_id') or os.environ.get('NDI_CLOUD_ORGANIZATION_ID'),
3434
'userId': kwargs.get('user_id'),
3535
'pageSize': kwargs.get('page_size', 20),
3636
'page': kwargs.get('page', 1)

‎src/ndi/cloud/internal/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
from .get_cloud_dataset_id_for_local_dataset import get_cloud_dataset_id_for_local_dataset
2+
from .get_weboptions_with_auth_header import get_weboptions_with_auth_header
3+
from .get_uploaded_document_ids import get_uploaded_document_ids
4+
from .get_uploaded_file_ids import get_uploaded_file_ids

‎src/ndi/cloud/internal/get_uploaded_document_ids.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
def get_uploaded_document_ids(dataset_id, verbose=False):
2+
"""
3+
Returns a dictionary of uploaded document IDs from the cloud.
4+
5+
Args:
6+
dataset_id (str): The cloud dataset ID.
7+
verbose (bool): Whether to print progress messages.
8+
9+
Returns:
10+
dict: A dictionary with keys 'ndiId' and 'apiId', containing lists of NDI and API document IDs.
11+
"""
12+
from ..api.documents import list_dataset_documents_all
13+
14+
if verbose:
15+
print(f'Fetching complete remote document list for dataset {dataset_id}...')
16+
17+
success, all_documents, _, _ = list_dataset_documents_all(dataset_id)
18+
19+
if not success:
20+
error_msg = all_documents.get('message', 'Unknown error') if isinstance(all_documents, dict) else all_documents
21+
raise RuntimeError(f"Failed to list remote documents: {error_msg}")
22+
23+
if not all_documents:
24+
if verbose:
25+
print('No remote documents found.')
26+
return {'ndiId': [], 'apiId': []}
27+
28+
all_ndi_ids = [doc.get('ndiId') for doc in all_documents]
29+
all_api_ids = [doc.get('id') for doc in all_documents]
30+
31+
if verbose:
32+
print(f'Total remote documents processed: {len(all_ndi_ids)}.')
33+
34+
return {'ndiId': all_ndi_ids, 'apiId': all_api_ids}

‎src/ndi/cloud/internal/get_uploaded_file_ids.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
def get_uploaded_file_ids(dataset_id, verbose=False):
2+
"""
3+
Returns a list of uploaded file IDs from the cloud.
4+
5+
Args:
6+
dataset_id (str): The cloud dataset ID.
7+
verbose (bool): Whether to print progress messages.
8+
9+
Returns:
10+
list: A list of unique file UIDs.
11+
"""
12+
from ..api.documents import list_dataset_documents_all
13+
14+
if verbose:
15+
print(f'Listing uploaded files for dataset {dataset_id}...')
16+
17+
success, all_documents, _, _ = list_dataset_documents_all(dataset_id)
18+
19+
if not success:
20+
error_msg = all_documents.get('message', 'Unknown error') if isinstance(all_documents, dict) else all_documents
21+
raise RuntimeError(f"Failed to list remote documents for files: {error_msg}")
22+
23+
file_uids = set()
24+
if all_documents:
25+
for document in all_documents:
26+
# Assuming document structure from cloud: properties are usually top-level or under 'document_properties' depending on API
27+
# Cloud usually returns the document object itself as dict
28+
# Check if 'files' property exists
29+
file_info = document.get('files', {}).get('file_info', [])
30+
for info in file_info:
31+
locations = info.get('locations', [])
32+
for location in locations:
33+
uid = location.get('uid')
34+
if uid:
35+
file_uids.add(uid)
36+
37+
if verbose:
38+
print(f'Found {len(file_uids)} unique file UIDs.')
39+
40+
return list(file_uids)

‎src/ndi/cloud/internal/get_weboptions_with_auth_header.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
def get_weboptions_with_auth_header(token=None):
2+
"""
3+
Returns the web options (headers) with the authentication header.
4+
5+
Args:
6+
token (str, optional): The authentication token. If not provided, it will be retrieved using authenticate().
7+
8+
Returns:
9+
dict: A dictionary containing the headers.
10+
"""
11+
if token is None:
12+
from ..authenticate import authenticate
13+
token = authenticate()
14+
15+
if not token:
16+
# Should we raise error?
17+
# Matlab might return empty or error.
18+
# For now, return empty headers or raise error?
19+
# authenticate() prints error if interactive login fails.
20+
pass
21+
22+
headers = {
23+
'Authorization': f'Bearer {token}',
24+
'Content-Type': 'application/json'
25+
}
26+
27+
return headers

‎src/ndi/cloud/sync/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
from .download_new_impl import download_new
22
from .sync_options import SyncOptions
33
from .enum.sync_mode import SyncMode
4+
from .mirror_from_remote import mirror_from_remote
5+
from .mirror_to_remote import mirror_to_remote
6+
from .two_way_sync import two_way_sync

0 commit comments

Comments
 (0)