From 3c75e973f2189b0f0b77ece28afadcbbaf754c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vil=C3=A9m=20Zouhar?= Date: Fri, 25 Jul 2025 19:03:22 -0700 Subject: [PATCH 1/5] create campaign status page for ESA --- Campaign/views.py | 125 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 33 deletions(-) diff --git a/Campaign/views.py b/Campaign/views.py index ce7cd863..6126ef8b 100644 --- a/Campaign/views.py +++ b/Campaign/views.py @@ -23,6 +23,7 @@ from EvalData.models import PairwiseAssessmentResult from EvalData.models import seconds_to_timedelta from EvalData.models import TASK_DEFINITIONS +from EvalData.models.direct_assessment_document import DirectAssessmentDocumentTask # pylint: disable=import-error @@ -51,19 +52,27 @@ def campaign_status(request, campaign_name, sort_key=2): _msg = 'Failure to identify campaign {0}'.format(campaign_name) return HttpResponse(_msg, content_type='text/plain') + try: + campaign_opts = campaign.campaignOptions.lower().split(";") + # may raise KeyError + result_type = RESULT_TYPE_BY_CLASS_NAME[campaign.get_campaign_type()] + except KeyError as exc: + LOGGER.debug( + f'Invalid campaign type {campaign.get_campaign_type()} for campaign {campaign.campaignName}' + ) + LOGGER.error(exc) + return HttpResponse( + 'Invalid campaign type for campaign {0}'.format(campaign.campaignName), + content_type='text/plain', + ) + + # special handling for ESA + if "esa" in campaign_opts: + return campaign_status_esa(campaign) + _out = [] for team in campaign.teams.all(): for user in team.members.all(): - try: - campaign_opts = campaign.campaignOptions.lower().split(";") - # may raise KeyError - result_type = RESULT_TYPE_BY_CLASS_NAME[campaign.get_campaign_type()] - except KeyError as exc: - LOGGER.debug( - f'Invalid campaign type {campaign.get_campaign_type()} for campaign {campaign.campaignName}' - ) - LOGGER.error(exc) - continue _data = result_type.objects.filter( createdBy=user, completed=True, task__campaign=campaign.id @@ -118,29 +127,6 @@ def campaign_status(request, campaign_name, sort_key=2): (x[0], x[1], -len(json.loads(x[2])), x[3], x[4], x[5], x[6]) for x in _data ] - elif "esa" in campaign_opts: - is_mqm_or_esa = True - _data = _data.values_list( - 'start_time', - 'end_time', - 'score', - 'item__itemID', - 'item__targetID', - 'item__itemType', - 'item__id', - 'item__documentID', - ) - # compute time override based on document times - import collections - - _time_pairs = collections.defaultdict(list) - for x in _data: - _time_pairs[x[7] + " ||| " + x[4]].append((x[0], x[1])) - _time_pairs = [ - (min([x[0] for x in doc_v]), max([x[1] for x in doc_v])) - for doc, doc_v in _time_pairs.items() - ] - _data = [(x[0], x[1], x[2], x[3], x[4], x[5], x[6]) for x in _data] else: _data = _data.values_list( 'start_time', @@ -245,6 +231,79 @@ def campaign_status(request, campaign_name, sort_key=2): return HttpResponse(u'\n'.join(_txt), content_type='text/plain') +def campaign_status_esa(campaign) -> str: + import collections + out_str = """ + + + + """ + out_str += "\n" + out_str += "\n" + + for team in campaign.teams.all(): + for user in team.members.all(): + if user.is_staff: + continue + + out_str += "" + _data = DirectAssessmentDocumentResult.objects.filter( + createdBy=user, completed=True, task__campaign=campaign.id + ) + + total_count = None + if _data: + _data_all = DirectAssessmentDocumentTask.objects.filter(campaign=campaign.id) + # brute-force try to find if any task has at least one item annotated by this user + for task in _data_all: + for item in task.items.all(): + item = DirectAssessmentDocumentResult.objects.filter( + item=item, createdBy=user + ).last() + if item: + total_count = task.items.count() + break + if total_count: + break + if total_count is None: + out_str += f"" + out_str += f"" + out_str += "" + out_str += "" + else: + if total_count == len(_data): + out_str += f"" + else: + out_str += f"" + out_str += f"" + first_modified = min([x.start_time for x in _data]) + last_modified = max([x.end_time for x in _data]) + out_str += f"" + out_str += f"" + + times = collections.defaultdict() + for item in _data: + times[(item.item.documentID, item.item.targetID)] = (item.start_time, item.end_time) + annotation_time = sum([b-a for a, b in times.values()]) + annotation_time = f'{int(floor(annotation_time / 3600)):0>2d}h{int(floor((annotation_time % 3600) / 60)):0>2d}m' + + annotation_time_upper = last_modified - first_modified + annotation_time_upper = f'{int(floor(annotation_time_upper / 3600)):0>2d}h{int(floor((annotation_time_upper % 3600) / 60)):0>2d}m' + + out_str += f"" + out_str += "\n" + + out_str += "
UsernameProgressFirst ModifiedLast ModifiedAnnotation Time
{user.username} 💤0%{user.username} ✅{user.username} 🛠️{len(_data)}/{total_count} ({len(_data) / total_count:.0%}){str(datetime(1970, 1, 1) + seconds_to_timedelta(first_modified)).split('.')[0]}{str(datetime(1970, 1, 1) + seconds_to_timedelta(last_modified)).split('.')[0]}{annotation_time} - {annotation_time_upper}
" + return HttpResponse(out_str, content_type='text/html') + + def stat_reliable_testing(_data, campaign_opts, result_type): _annotations = len(set([x[6] for x in _data])) _user_mean = sum([x[2] for x in _data]) / (_annotations or 1) From 55715c817da422d5bcd7e3716e21d8e9b9f16298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vil=C3=A9m=20Zouhar?= Date: Fri, 25 Jul 2025 19:25:29 -0700 Subject: [PATCH 2/5] update campaign-status styling --- Campaign/views.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Campaign/views.py b/Campaign/views.py index 6126ef8b..616dce54 100644 --- a/Campaign/views.py +++ b/Campaign/views.py @@ -243,8 +243,12 @@ def campaign_status_esa(campaign) -> str: td, th { padding: 5px; } - + * { + font-family: monospace; + } + \n """ + out_str += f"

{campaign.campaignName}

\n" out_str += "\n" out_str += "\n" @@ -252,12 +256,10 @@ def campaign_status_esa(campaign) -> str: for user in team.members.all(): if user.is_staff: continue - out_str += "" _data = DirectAssessmentDocumentResult.objects.filter( createdBy=user, completed=True, task__campaign=campaign.id ) - total_count = None if _data: _data_all = DirectAssessmentDocumentTask.objects.filter(campaign=campaign.id) @@ -274,7 +276,7 @@ def campaign_status_esa(campaign) -> str: break if total_count is None: out_str += f"" - out_str += f"" + out_str += "" out_str += "" out_str += "" else: @@ -285,8 +287,15 @@ def campaign_status_esa(campaign) -> str: out_str += f"" first_modified = min([x.start_time for x in _data]) last_modified = max([x.end_time for x in _data]) - out_str += f"" - out_str += f"" + + first_modified_str = str(datetime(1970, 1, 1) + seconds_to_timedelta(first_modified)).split('.')[0] + last_modified_str = str(datetime(1970, 1, 1) + seconds_to_timedelta(last_modified)).split('.')[0] + # remove seconds + first_modified_str = ":".join(first_modified_str.split(":")[:-1]) + last_modified_str = ":".join(last_modified_str.split(":")[:-1]) + + out_str += f"" + out_str += f"" times = collections.defaultdict() for item in _data: From ed89e7e4d3c1aa351f8ab476bc12cd1936a752e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vil=C3=A9m=20Zouhar?= Date: Fri, 25 Jul 2025 19:43:13 -0700 Subject: [PATCH 3/5] speed-up campaign-status and next document fetch --- Campaign/views.py | 18 +++--------------- EvalData/models/direct_assessment_document.py | 11 +++++++---- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/Campaign/views.py b/Campaign/views.py index 616dce54..a37b8db0 100644 --- a/Campaign/views.py +++ b/Campaign/views.py @@ -260,26 +260,14 @@ def campaign_status_esa(campaign) -> str: _data = DirectAssessmentDocumentResult.objects.filter( createdBy=user, completed=True, task__campaign=campaign.id ) - total_count = None - if _data: - _data_all = DirectAssessmentDocumentTask.objects.filter(campaign=campaign.id) - # brute-force try to find if any task has at least one item annotated by this user - for task in _data_all: - for item in task.items.all(): - item = DirectAssessmentDocumentResult.objects.filter( - item=item, createdBy=user - ).last() - if item: - total_count = task.items.count() - break - if total_count: - break - if total_count is None: + if not _data: out_str += f"" out_str += "" out_str += "" out_str += "" else: + task = DirectAssessmentDocumentTask.objects.filter(id=_data[0].task_id).first() + total_count = task.items.count() if total_count == len(_data): out_str += f"" else: diff --git a/EvalData/models/direct_assessment_document.py b/EvalData/models/direct_assessment_document.py index 4499bf6a..2a3a56dd 100644 --- a/EvalData/models/direct_assessment_document.py +++ b/EvalData/models/direct_assessment_document.py @@ -263,13 +263,16 @@ def next_document_for_user_mqmesa(self, user): doc_items_results, """ - # get all items (100) and try to find resul + # get all items and try to find a matching result + # TODO: probably can be optimized better + + items_user = DirectAssessmentDocumentResult.objects.filter( + activated=False, completed=True, createdBy=user + ) all_items = [ ( item, - DirectAssessmentDocumentResult.objects.filter( - item=item, activated=False, completed=True, createdBy=user - ).last(), + items_user.filter(item=item).last(), ) for item in self.items.all().order_by('id') ] From 7e8b9fa2321d12a3a2bf8b721e0be2af4d77257b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vil=C3=A9m=20Zouhar?= Date: Fri, 25 Jul 2025 20:03:56 -0700 Subject: [PATCH 4/5] edit style of times --- Campaign/views.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/Campaign/views.py b/Campaign/views.py index a37b8db0..e112e0e9 100644 --- a/Campaign/views.py +++ b/Campaign/views.py @@ -250,7 +250,9 @@ def campaign_status_esa(campaign) -> str: """ out_str += f"

{campaign.campaignName}

\n" out_str += "
UsernameProgressFirst ModifiedLast ModifiedAnnotation Time
{user.username} 💤0%{len(_data)}/{total_count} ({len(_data) / total_count:.0%}){str(datetime(1970, 1, 1) + seconds_to_timedelta(first_modified)).split('.')[0]}{str(datetime(1970, 1, 1) + seconds_to_timedelta(last_modified)).split('.')[0]}{first_modified_str}{last_modified_str}{user.username} 💤{user.username} ✅
\n" - out_str += "\n" + out_str += "" + "".join( + f"" for x in ["Username", "Progress", "First Modified", "Last Modified", "Time (Last-First)", "Time (Real)"] + ) + "\n" for team in campaign.teams.all(): for user in team.members.all(): @@ -265,6 +267,8 @@ def campaign_status_esa(campaign) -> str: out_str += "" out_str += "" out_str += "" + out_str += "" + out_str += "" else: task = DirectAssessmentDocumentTask.objects.filter(id=_data[0].task_id).first() total_count = task.items.count() @@ -284,17 +288,23 @@ def campaign_status_esa(campaign) -> str: out_str += f"" out_str += f"" + annotation_time_upper = last_modified - first_modified + annotation_time_upper = f'{int(floor(annotation_time_upper / 3600)):0>2d}h {int(floor((annotation_time_upper % 3600) / 60)):0>2d}m' + out_str += f"" - times = collections.defaultdict() + times = collections.defaultdict(list) for item in _data: - times[(item.item.documentID, item.item.targetID)] = (item.start_time, item.end_time) - annotation_time = sum([b-a for a, b in times.values()]) - annotation_time = f'{int(floor(annotation_time / 3600)):0>2d}h{int(floor((annotation_time % 3600) / 60)):0>2d}m' + times[(item.item.documentID, item.item.targetID)].append((item.start_time, item.end_time)) + times = [ + (min([x[0] for x in doc_v]), max([x[1] for x in doc_v])) + for doc, doc_v in times.items() + ] - annotation_time_upper = last_modified - first_modified - annotation_time_upper = f'{int(floor(annotation_time_upper / 3600)):0>2d}h{int(floor((annotation_time_upper % 3600) / 60)):0>2d}m' + annotation_time = sum([b-a for a, b in times]) + annotation_time = f'{int(floor(annotation_time / 3600)):0>2d}h {int(floor((annotation_time % 3600) / 60)):0>2d}m' + + out_str += f"" - out_str += f"" out_str += "\n" out_str += "
UsernameProgressFirst ModifiedLast ModifiedAnnotation Time
{x}
{first_modified_str}{last_modified_str}{annotation_time_upper}{annotation_time}{annotation_time} - {annotation_time_upper}
" From 11cd25995bccf85bd9e5f01cc205827c968f47e3 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sat, 26 Jul 2025 09:45:42 -0700 Subject: [PATCH 5/5] use TaskAgenda to get the task for a user; display 0/xxx even if no annotations made --- Campaign/views.py | 58 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/Campaign/views.py b/Campaign/views.py index e112e0e9..36c7f927 100644 --- a/Campaign/views.py +++ b/Campaign/views.py @@ -23,6 +23,7 @@ from EvalData.models import PairwiseAssessmentResult from EvalData.models import seconds_to_timedelta from EvalData.models import TASK_DEFINITIONS +from EvalData.models import TaskAgenda from EvalData.models.direct_assessment_document import DirectAssessmentDocumentTask # pylint: disable=import-error @@ -65,7 +66,7 @@ def campaign_status(request, campaign_name, sort_key=2): 'Invalid campaign type for campaign {0}'.format(campaign.campaignName), content_type='text/plain', ) - + # special handling for ESA if "esa" in campaign_opts: return campaign_status_esa(campaign) @@ -237,7 +238,7 @@ def campaign_status_esa(campaign) -> str: