Skip to content

[codex] Guard data location relinks by session IDs #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ehennestad wants to merge 1 commit into
base: dev
Choose a base branch
from
+224 −130
Draft

[codex] Guard data location relinks by session IDs #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 34 additions & 3 deletions code/+nansen/+config/+dloc/@DataLocationModelApp/DataLocationModelApp.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@

properties
DataLocationModel
CommitValidator = []
end

properties (Access = private)
Expand Down Expand Up @@ -228,17 +229,31 @@ function createUIModules(obj, moduleNumber, varargin)

switch selection
case 'Yes'
updatedModuleIdx = [];
for i = 2:3
if ~isempty(obj.UIModule{i})
d = uiprogressdlg(obj.Figure, 'Title','Updading Model',...
d = uiprogressdlg(obj.Figure, 'Title','Updating Model',...
'Message', 'Updating paths for session folders of Datalocations...', ...
'Indeterminate','on');
closeDialog = onCleanup(@() close(d));
obj.UIModule{i}.updateDataLocationModel()
close(d)
obj.UIModule{i}.markClean()
updatedModuleIdx = [updatedModuleIdx, i]; %#ok<AGROW>
clear closeDialog
end
end

try
obj.runCommitValidator()
catch exception
obj.alertCommitValidationFailed(exception)
doAbort = true;
return
end

obj.DataLocationModel.save()
for i = updatedModuleIdx
obj.UIModule{i}.markClean()
end
obj.UIModule{1}.markClean()
evtData = event.EventData;
obj.notify('DataLocationModelChanged', evtData)
Expand All @@ -252,6 +267,22 @@ function createUIModules(obj, moduleNumber, varargin)
end
end
end

function runCommitValidator(obj)
if isempty(obj.CommitValidator)
return
end
obj.CommitValidator(obj.DataLocationModel);
end

function alertCommitValidationFailed(obj, exception)
title = 'Data Location Update Aborted';
if ~isempty(obj.Figure) && isvalid(obj.Figure)
uialert(obj.Figure, exception.message, title, 'Icon', 'warning')
else
warning(exception.identifier, '%s', exception.message)
end
end
end

methods % Get methods
Expand Down
131 changes: 5 additions & 126 deletions code/+nansen/+manage/updateSessionDatalocations.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,11 @@

% Todo: Should this be a method of the model?

import nansen.dataio.session.listSessionFolders
import nansen.dataio.session.matchSessionFolders

% % Use the folder structure to detect session folders.
detectedSessionFolders = listSessionFolders(dataLocationModel, 'all');
try
[sessionFolders, sessionIDs] = matchSessionFolders(dataLocationModel, detectedSessionFolders);
catch exception
switch exception.identifier
case 'NANSEN:DataIO:NoMatchingSessionFolders'
error('NANSEN:DataLocations:NoMatchingSessionFolders', ...
['No session folders were detected using the current ', ...
'DataLocationModel. Please check the DataLocationModel configuration.'])
otherwise
rethrow(exception)
end
end
sessionIDs = normalizeSessionIDList(sessionIDs);

tableSessionIDs = getTableSessionIDs(sessionTable);
if ~isempty(sessionIDs) && ~isempty(tableSessionIDs) ...
&& ~any(ismember(sessionIDs, tableSessionIDs))
error('NANSEN:DataLocations:NoMatchingTableSessionIDs', '%s', ...
createNoMatchingSessionIDsError(...
dataLocationModel, detectedSessionFolders, sessionIDs, tableSessionIDs))
end
report = nansen.manage.validateDataLocationRelink(...
sessionTable, dataLocationModel);
sessionFolders = report.MatchedSessionFolders;
sessionIDs = report.ExtractedSessionIDs;
tableSessionIDs = report.TableSessionIDs;

% Match sessions in table with sessionFolder (data locations)
unresolvedIdx = [];
Expand Down Expand Up @@ -73,103 +52,3 @@
dataLocationStructs_ = mat2cell(dataLocationStructs, ones(siz_(1),1), siz_(2));
sessionTable.replaceDataColumn('DataLocation', dataLocationStructs_ );
end

function sessionIDs = getTableSessionIDs(sessionTable)
sessionIDs = sessionTable.entries{:, 'sessionID'};
sessionIDs = normalizeSessionIDList(sessionIDs);
end

function sessionIDs = normalizeSessionIDList(sessionIDs)
if iscell(sessionIDs)
sessionIDs = sessionIDs(:);
for i = 1:numel(sessionIDs)
sessionIDs{i} = convertSessionIDToChar(sessionIDs{i});
end
elseif isstring(sessionIDs)
sessionIDs = cellstr(sessionIDs(:));
elseif ischar(sessionIDs)
sessionIDs = cellstr(sessionIDs);
else
sessionIDs = arrayfun(@convertSessionIDToChar, sessionIDs(:), ...
'UniformOutput', false);
end

sessionIDs = sessionIDs(:)';
end

function sessionID = convertSessionIDToChar(sessionID)
if iscell(sessionID)
if isempty(sessionID)
sessionID = '';
else
sessionID = convertSessionIDToChar(sessionID{1});
end
elseif isstring(sessionID)
if isempty(sessionID)
sessionID = '';
else
sessionID = char(sessionID);
end
elseif ischar(sessionID)
return
elseif isnumeric(sessionID)
sessionID = num2str(sessionID);
else
sessionID = char(string(sessionID));
end
end

function message = createNoMatchingSessionIDsError(...
dataLocationModel, sessionFolderList, sessionIDs, tableSessionIDs)

header = createDetectedFoldersMessage(dataLocationModel, sessionFolderList);
message = sprintf('%s\n\nFirst extracted IDs:\n%s\n\nFirst table IDs:\n%s', ...
header, formatSessionIDPreview(sessionIDs), ...
formatSessionIDPreview(tableSessionIDs));
end

function message = createDetectedFoldersMessage(dataLocationModel, sessionFolderList)
dataLocationNames = {dataLocationModel.Data.Name};
folderCounts = zeros(1, numel(dataLocationNames));

for i = 1:numel(dataLocationNames)
if isfield(sessionFolderList, dataLocationNames{i})
folderCounts(i) = numel(sessionFolderList.(dataLocationNames{i}));
end
end

detectedDataLocationIdx = find(folderCounts > 0);
if numel(detectedDataLocationIdx) == 1
idx = detectedDataLocationIdx;
message = sprintf(['Detected %d session folders for "%s", ', ...
'but none matched the table session IDs.'], ...
folderCounts(idx), dataLocationNames{idx});
else
message = sprintf(['Detected session folders, but none matched ', ...
'the table session IDs.\n\nDetected folder counts:\n%s'], ...
formatDetectedFolderCounts(dataLocationNames, folderCounts));
end
end

function countList = formatDetectedFolderCounts(dataLocationNames, folderCounts)
countLines = cell(1, numel(dataLocationNames));
for i = 1:numel(dataLocationNames)
countLines{i} = sprintf(' %s: %d', ...
dataLocationNames{i}, folderCounts(i));
end
countList = strjoin(countLines, sprintf('\n'));
end

function preview = formatSessionIDPreview(sessionIDs)
numPreview = min(2, numel(sessionIDs));
if numPreview == 0
preview = ' <none>';
return
end

previewLines = cell(1, numPreview);
for i = 1:numPreview
previewLines{i} = sprintf(' %s', sessionIDs{i});
end
preview = strjoin(previewLines, sprintf('\n'));
end
159 changes: 159 additions & 0 deletions code/+nansen/+manage/validateDataLocationRelink.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
function report = validateDataLocationRelink(sessionTable, dataLocationModel)
%validateDataLocationRelink Validate in-place datalocation relinking.
%
% report = validateDataLocationRelink(sessionTable, dataLocationModel)
% verifies that session ids extracted with the given DataLocationModel
% overlap with the immutable session ids in the existing session table.

import nansen.dataio.session.listSessionFolders
import nansen.dataio.session.matchSessionFolders

assertSessionIDColumnExists(sessionTable)

detectedSessionFolders = listSessionFolders(dataLocationModel, 'all');
try
[sessionFolders, sessionIDs] = matchSessionFolders(...
dataLocationModel, detectedSessionFolders);
catch exception
switch exception.identifier
case 'NANSEN:DataIO:NoMatchingSessionFolders'
error('NANSEN:DataLocations:NoMatchingSessionFolders', ...
['No session folders were detected using the current ', ...
'DataLocationModel. Please check the DataLocationModel configuration.'])
otherwise
rethrow(exception)
end
end

sessionIDs = normalizeSessionIDList(sessionIDs);
tableSessionIDs = getTableSessionIDs(sessionTable);
matchingSessionIDs = intersect(sessionIDs, tableSessionIDs, 'stable');

report = struct();
report.DetectedSessionFolders = detectedSessionFolders;
report.MatchedSessionFolders = sessionFolders;
report.ExtractedSessionIDs = sessionIDs;
report.TableSessionIDs = tableSessionIDs;
report.MatchingSessionIDs = matchingSessionIDs;
report.NumMatchingSessionIDs = numel(matchingSessionIDs);

if ~isempty(sessionIDs) && ~isempty(tableSessionIDs) ...
&& isempty(matchingSessionIDs)
error('NANSEN:DataLocations:SessionIdentityMismatch', '%s', ...
createSessionIdentityMismatchError(...
dataLocationModel, detectedSessionFolders, sessionIDs, tableSessionIDs))
end
end

function assertSessionIDColumnExists(sessionTable)
tableVariableNames = sessionTable.entries.Properties.VariableNames;
if ~any(strcmp(tableVariableNames, 'sessionID'))
error('NANSEN:DataLocations:MissingSessionIDColumn', ...
['Can not update data locations because the session table ', ...
'does not have a sessionID column.'])
end
end

function sessionIDs = getTableSessionIDs(sessionTable)
sessionIDs = sessionTable.entries{:, 'sessionID'};
sessionIDs = normalizeSessionIDList(sessionIDs);
end

function sessionIDs = normalizeSessionIDList(sessionIDs)
if iscell(sessionIDs)
sessionIDs = sessionIDs(:);
for i = 1:numel(sessionIDs)
sessionIDs{i} = convertSessionIDToChar(sessionIDs{i});
end
elseif isstring(sessionIDs)
sessionIDs = cellstr(sessionIDs(:));
elseif ischar(sessionIDs)
sessionIDs = cellstr(sessionIDs);
else
sessionIDs = arrayfun(@convertSessionIDToChar, sessionIDs(:), ...
'UniformOutput', false);
end

sessionIDs = sessionIDs(:)';
end

function sessionID = convertSessionIDToChar(sessionID)
if iscell(sessionID)
if isempty(sessionID)
sessionID = '';
else
sessionID = convertSessionIDToChar(sessionID{1});
end
elseif isstring(sessionID)
if isempty(sessionID)
sessionID = '';
else
sessionID = char(sessionID);
end
elseif ischar(sessionID)
return
elseif isnumeric(sessionID)
sessionID = num2str(sessionID);
else
sessionID = char(string(sessionID));
end
end

function message = createSessionIdentityMismatchError(...
dataLocationModel, sessionFolderList, sessionIDs, tableSessionIDs)

header = createDetectedFoldersMessage(dataLocationModel, sessionFolderList);
instruction = ['Session IDs are immutable for in-place data location ', ...
'updates. If the Session ID definition was changed intentionally, ', ...
'run nansen.configureProject to rebuild the session table.'];

message = sprintf('%s\n\nFirst extracted IDs:\n%s\n\nFirst table IDs:\n%s\n\n%s', ...
header, formatSessionIDPreview(sessionIDs), ...
formatSessionIDPreview(tableSessionIDs), instruction);
end

function message = createDetectedFoldersMessage(dataLocationModel, sessionFolderList)
dataLocationNames = {dataLocationModel.Data.Name};
folderCounts = zeros(1, numel(dataLocationNames));

for i = 1:numel(dataLocationNames)
if isfield(sessionFolderList, dataLocationNames{i})
folderCounts(i) = numel(sessionFolderList.(dataLocationNames{i}));
end
end

detectedDataLocationIdx = find(folderCounts > 0);
if numel(detectedDataLocationIdx) == 1

Check notice

Code scanning / Code Analyzer

To improve performance, use 'isscalar' instead of length comparison. Note

To improve performance, use 'isscalar' instead of length comparison.
idx = detectedDataLocationIdx;
message = sprintf(['Detected %d session folders for "%s", ', ...
'but none matched the table session IDs.'], ...
folderCounts(idx), dataLocationNames{idx});
else
message = sprintf(['Detected session folders, but none matched ', ...
'the table session IDs.\n\nDetected folder counts:\n%s'], ...
formatDetectedFolderCounts(dataLocationNames, folderCounts));
end
end

function countList = formatDetectedFolderCounts(dataLocationNames, folderCounts)
countLines = cell(1, numel(dataLocationNames));
for i = 1:numel(dataLocationNames)
countLines{i} = sprintf(' %s: %d', ...
dataLocationNames{i}, folderCounts(i));
end
countList = strjoin(countLines, sprintf('\n'));

Check notice

Code scanning / Code Analyzer

For readability, consider using the 'newline' function instead of 'sprintf('\n')'. Note

For readability, consider using the 'newline' function instead of 'sprintf('\n')'.
end

function preview = formatSessionIDPreview(sessionIDs)
numPreview = min(2, numel(sessionIDs));
if numPreview == 0
preview = ' <none>';
return
end

previewLines = cell(1, numPreview);
for i = 1:numPreview
previewLines{i} = sprintf(' %s', sessionIDs{i});
end
preview = strjoin(previewLines, sprintf('\n'));

Check notice

Code scanning / Code Analyzer

For readability, consider using the 'newline' function instead of 'sprintf('\n')'. Note

For readability, consider using the 'newline' function instead of 'sprintf('\n')'.
end
Loading
Loading