From 0982d912c9cae84266fc5ce48a8906e893fb3b79 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Fri, 18 Jul 2025 17:46:52 -0400 Subject: [PATCH 01/20] Refactor redata_reports to enable code reuse with new reports --- .gitignore | 2 +- deploy.sh | 2 +- .../redata_reports/run => lib}/functions.py | 2 +- .../run => lib}/secrets.example.py | 17 +++++++---------- packages/redata_reports/run/build.sh | 1 + packages/redata_reports/run/items_report.py | 11 +++++++---- packages/redata_reports/run/main.py | 13 ++++++++----- packages/redata_reports/run/userquota_report.py | 9 ++++++--- packages/redata_reports/run/version.py | 2 +- project.yml | 4 ++-- 10 files changed, 35 insertions(+), 28 deletions(-) rename {packages/redata_reports/run => lib}/functions.py (99%) rename {packages/redata_reports/run => lib}/secrets.example.py (59%) diff --git a/.gitignore b/.gitignore index 22d81ec..bc33c81 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ lib64/ parts/ sdist/ diff --git a/deploy.sh b/deploy.sh index 71b8fb9..b79afcb 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,3 +1,3 @@ #!/bin/bash -doctl serverless deploy . --remote-build --env packages/redata_reports/run/secrets.py \ No newline at end of file +doctl serverless deploy . --remote-build --env lib/secrets.py \ No newline at end of file diff --git a/packages/redata_reports/run/functions.py b/lib/functions.py similarity index 99% rename from packages/redata_reports/run/functions.py rename to lib/functions.py index 53f1faa..b0aea7d 100644 --- a/packages/redata_reports/run/functions.py +++ b/lib/functions.py @@ -11,7 +11,7 @@ def get_request_headers(): Returns the headers to use for the figshare requests.get calls """ headers = {'Content-Type': 'application/json', - 'Authorization': 'Bearer {0}'.format(environ['API_TOKEN'])} + 'Authorization': 'Bearer {0}'.format(environ['API_FIGSHARE_TOKEN'])} return headers diff --git a/packages/redata_reports/run/secrets.example.py b/lib/secrets.example.py similarity index 59% rename from packages/redata_reports/run/secrets.example.py rename to lib/secrets.example.py index 3c13931..f4efae2 100644 --- a/packages/redata_reports/run/secrets.example.py +++ b/lib/secrets.example.py @@ -5,8 +5,13 @@ from os import environ # Figshare API -api_url_base = '' -api_token = '' +api_figshare_url_base = '' +api_figshare_token = '' + +# Trello API +api_trello_url_base = '' +api_trello_key = '' +api_trello_token = '' # Google sheet gsheets_dashboard_post_url = '' @@ -14,11 +19,3 @@ # DO access token. When deployed as a function, all requests must send this token in the 't' parameter of the GET request do_token = '' - - -# ************************************************************** -environ['API_URL_BASE'] = api_url_base -environ['API_TOKEN'] = api_token -environ['GSHEETS_DASHBOARD_POST_URL'] = gsheets_dashboard_post_url -environ['GSHEETS_DASHBOARD_KEY'] = gsheets_dashboard_key -environ['TOKEN'] = do_token diff --git a/packages/redata_reports/run/build.sh b/packages/redata_reports/run/build.sh index 4da805b..5982300 100644 --- a/packages/redata_reports/run/build.sh +++ b/packages/redata_reports/run/build.sh @@ -7,4 +7,5 @@ set -e virtualenv virtualenv source virtualenv/bin/activate pip install -r requirements.txt +cp ../../../lib/*.py . deactivate \ No newline at end of file diff --git a/packages/redata_reports/run/items_report.py b/packages/redata_reports/run/items_report.py index 90c27ae..a97bf0c 100644 --- a/packages/redata_reports/run/items_report.py +++ b/packages/redata_reports/run/items_report.py @@ -4,11 +4,14 @@ # # Author: Fernando Rios +import sys import requests import simplejson as json from multiprocessing import Pool from datetime import datetime from os import environ + +sys.path.insert(0, 'lib/') import functions as f @@ -16,7 +19,7 @@ def get_institution_articles(): page = 1 article_list = [] while True: - api_url = '{0}/account/institution/articles?page={1}&page_size=1000'.format(environ['API_URL_BASE'], page) + api_url = '{0}/account/institution/articles?page={1}&page_size=1000'.format(environ['API_FIGSHARE_URL_BASE'], page) page += 1 response = requests.get(api_url, headers=f.get_request_headers()) @@ -37,14 +40,14 @@ def get_public_article_info(article_ids): article_ids = [article_ids] for id in article_ids: - api_url = '{0}/articles/{1}/versions'.format(environ['API_URL_BASE'], id) + api_url = '{0}/articles/{1}/versions'.format(environ['API_FIGSHARE_URL_BASE'], id) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: article_versions = json.loads(response.text) if len(article_versions) > 0: for version in article_versions: version_num = version['version'] - api_url = '{0}/articles/{1}/versions/{2}'.format(environ['API_URL_BASE'], id, version_num) + api_url = '{0}/articles/{1}/versions/{2}'.format(environ['API_FIGSHARE_URL_BASE'], id, version_num) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: @@ -69,7 +72,7 @@ def get_private_article_info(article_ids): article_ids = [article_ids] for id in article_ids: - api_url = '{0}/account/articles/{1}'.format(environ['API_URL_BASE'], id) + api_url = '{0}/account/articles/{1}'.format(environ['API_FIGSHARE_URL_BASE'], id) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: article_info.append(json.loads(response.text)) diff --git a/packages/redata_reports/run/main.py b/packages/redata_reports/run/main.py index 417d7c8..d3cb37a 100644 --- a/packages/redata_reports/run/main.py +++ b/packages/redata_reports/run/main.py @@ -5,13 +5,16 @@ # Author: Fernando Rios import argparse +import sys from os import environ from version import __version__, __commit__ -import secrets -import functions as f import items_report import userquota_report +sys.path.insert(0, 'lib/') +import functions as f +import secrets + def init_argparse(): parser = argparse.ArgumentParser( @@ -36,7 +39,7 @@ def init_argparse(): ) parser.add_argument( '-u', '--units', choices=['B', 'KB', 'MB', 'GB', 'TB'], - default='GB', help='Set the output units. Default is %(default)s') + default='GB', help='Set the output size units. Default is %(default)s') return parser @@ -71,8 +74,8 @@ def run(args): if __name__ == '__main__': args = init_argparse().parse_args() - environ['API_URL_BASE'] = secrets.api_url_base - environ['API_TOKEN'] = secrets.api_token + environ['API_FIGSHARE_URL_BASE'] = secrets.api_figshare_url_base + environ['API_FIGSHARE_TOKEN'] = secrets.api_figshare_token environ['GSHEETS_DASHBOARD_POST_URL'] = secrets.gsheets_dashboard_post_url environ['GSHEETS_DASHBOARD_KEY'] = secrets.gsheets_dashboard_key environ['TOKEN'] = secrets.do_token diff --git a/packages/redata_reports/run/userquota_report.py b/packages/redata_reports/run/userquota_report.py index c071865..4624263 100644 --- a/packages/redata_reports/run/userquota_report.py +++ b/packages/redata_reports/run/userquota_report.py @@ -8,10 +8,13 @@ # # Author: Fernando Rios +import sys import requests import simplejson as json from multiprocessing import Pool from os import environ + +sys.path.insert(0, 'lib/') import functions as f @@ -19,7 +22,7 @@ def get_institution_accounts(): page = 1 accounts_list = [] while True: - api_url = '{0}/account/institution/accounts?page={1}&page_size=1000'.format(environ['API_URL_BASE'], page) + api_url = '{0}/account/institution/accounts?page={1}&page_size=1000'.format(environ['API_FIGSHARE_URL_BASE'], page) page += 1 response = requests.get(api_url, headers=f.get_request_headers()) @@ -33,14 +36,14 @@ def get_institution_accounts(): def get_account_info(id): - api_url = '{0}/account?impersonate={1}'.format(environ['API_URL_BASE'], id) + api_url = '{0}/account?impersonate={1}'.format(environ['API_FIGSHARE_URL_BASE'], id) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: data = json.loads(response.text) elif response.status_code == 403: # can't impersonate ourselves - api_url = '{0}/account'.format(environ['API_URL_BASE']) + api_url = '{0}/account'.format(environ['API_FIGSHARE_URL_BASE']) response = requests.get(api_url, headers=f.get_request_headers()) data = json.loads(response.text) else: diff --git a/packages/redata_reports/run/version.py b/packages/redata_reports/run/version.py index 6237c39..9c86b93 100644 --- a/packages/redata_reports/run/version.py +++ b/packages/redata_reports/run/version.py @@ -14,4 +14,4 @@ def get_commit(repo_path): __commit__ = get_commit('.') -__version__ = '1.1.1' +__version__ = '1.1.2' diff --git a/project.yml b/project.yml index 0589014..92ee069 100644 --- a/project.yml +++ b/project.yml @@ -8,8 +8,8 @@ packages: timeout: 420000 memory: 512 environment: - API_URL_BASE: ${api_url_base} - API_TOKEN: ${api_token} + API_FIGSHARE_URL_BASE: ${api_figshare_url_base} + API_FIGSHARE_TOKEN: ${api_figshare_token} GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} TOKEN: ${do_token} \ No newline at end of file From 66f22592df12243ac673d3baa41ae8f859cc4a45 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Fri, 18 Jul 2025 17:51:32 -0400 Subject: [PATCH 02/20] Add working stub functions for Trello-based reports Works locally and in DO. Outputs the current date --- packages/trello_reports/run/.ignore | 2 + packages/trello_reports/run/__main__.py | 26 +++++++ packages/trello_reports/run/build.sh | 11 +++ packages/trello_reports/run/main.py | 80 ++++++++++++++++++++ packages/trello_reports/run/requirements.txt | 2 + packages/trello_reports/run/version.py | 17 +++++ project.yml | 15 ++++ 7 files changed, 153 insertions(+) create mode 100644 packages/trello_reports/run/.ignore create mode 100644 packages/trello_reports/run/__main__.py create mode 100644 packages/trello_reports/run/build.sh create mode 100644 packages/trello_reports/run/main.py create mode 100644 packages/trello_reports/run/requirements.txt create mode 100644 packages/trello_reports/run/version.py diff --git a/packages/trello_reports/run/.ignore b/packages/trello_reports/run/.ignore new file mode 100644 index 0000000..585854c --- /dev/null +++ b/packages/trello_reports/run/.ignore @@ -0,0 +1,2 @@ +# These files will not be included in the DO deployed package +secrets* \ No newline at end of file diff --git a/packages/trello_reports/run/__main__.py b/packages/trello_reports/run/__main__.py new file mode 100644 index 0000000..a6cebbb --- /dev/null +++ b/packages/trello_reports/run/__main__.py @@ -0,0 +1,26 @@ +# Entry point for DO functions + +import main as m +from os import environ + + +def main(event, context): + # Only allow authenticated requests + accesstoken = event.get("t", "") + if accesstoken != environ["TOKEN"]: + return { + "headers": {"Content-Type": "text/html"}, + "statusCode": 403, + "body": "Forbidden. Invalid token." + } + + args = m.init_argparse().parse_args() + args.sync_to_dashboard = True + + result = m.run(args) + + return { + "headers": {"Content-Type": "text/html"}, + "statusCode": 200, + "body": f"{result}" + } diff --git a/packages/trello_reports/run/build.sh b/packages/trello_reports/run/build.sh new file mode 100644 index 0000000..5982300 --- /dev/null +++ b/packages/trello_reports/run/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Script for remote DO builds + +set -e + +virtualenv virtualenv +source virtualenv/bin/activate +pip install -r requirements.txt +cp ../../../lib/*.py . +deactivate \ No newline at end of file diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py new file mode 100644 index 0000000..40056a5 --- /dev/null +++ b/packages/trello_reports/run/main.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# Runs various reports for ReDATA +# +# Author: Fernando Rios + +import argparse +import sys +from os import environ +from version import __version__, __commit__ +#import items_report + +sys.path.insert(0, 'lib/') +import functions as f +import secrets + +def init_argparse(): + parser = argparse.ArgumentParser( + description='Runs reports for Trello stats' + ) + parser.add_argument( + '-r', '--report', required=False, action='append', choices=['curators', 'items'], + help="Selects which report to run. This option can appear more than once" + ) + parser.add_argument( + '--sync-to-dashboard', required=False, action='store_true', + help="Uploads report data to the Google dashboard. Additionally, sets these flags: -u B -r items -r users" + ) + parser.add_argument( + '-v', '--version', action='version', + version=f'{parser.prog} v{__version__} {__commit__}' + ) + parser.add_argument( + '-o', '--outfile', metavar='PATH', nargs='?', + const=f"$$*$${f.get_report_date().strftime('%Y-%m-%dT%H%M%S')}.csv", + type=str, help="Write output to a file. If PATH isn't specified, file will default to a timestamped file in the current directory" + ) + parser.add_argument( + '-u', '--units', choices=['M', 'W', 'D', 'H', 'm', 's'], + default='H', help='Set the output time units. Default is %(default)s') + + return parser + + +def run(args): + print(f'This is trello-reports version v{__version__} {__commit__}') + + if args.sync_to_dashboard: + args.units = 's' + args.report = ['curators', 'items'] + + if not args.report: + return 'No report specified. Doing nothing' + + result = '' + if 'curators' in args.report: + print('Running "curators" report') + result = f.get_report_date() + #data = curator_report.run(args) + #result = result + f'Running "curator" report completed. {len(data)} curators.' + #if args.sync_to_dashboard: + # result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' + if 'items' in args.report: + print('Running "items" report') + result = 'not implemented' + + return result + + +if __name__ == '__main__': + args = init_argparse().parse_args() + + environ['API_TRELLO_URL_BASE'] = secrets.api_trello_url_base + environ['API_TRELLO_KEY'] = secrets.api_trello_key + environ['API_TRELLO_TOKEN'] = secrets.api_trello_token + environ['GSHEETS_DASHBOARD_POST_URL'] = secrets.gsheets_dashboard_post_url + environ['GSHEETS_DASHBOARD_KEY'] = secrets.gsheets_dashboard_key + environ['TOKEN'] = secrets.do_token + + print(run(args)) diff --git a/packages/trello_reports/run/requirements.txt b/packages/trello_reports/run/requirements.txt new file mode 100644 index 0000000..4104bca --- /dev/null +++ b/packages/trello_reports/run/requirements.txt @@ -0,0 +1,2 @@ +requests +simplejson \ No newline at end of file diff --git a/packages/trello_reports/run/version.py b/packages/trello_reports/run/version.py new file mode 100644 index 0000000..cea651d --- /dev/null +++ b/packages/trello_reports/run/version.py @@ -0,0 +1,17 @@ +from pathlib import Path + + +def get_commit(repo_path): + # https://stackoverflow.com/a/68215738 + git_folder = Path(repo_path, '../../../.git') + try: + head_name = Path(git_folder, 'HEAD').read_text().split('\n')[0].split(' ')[-1] + head_ref = Path(git_folder, head_name) + commit = head_ref.read_text().replace('\n', '') + except Exception: + return '' + return commit + + +__commit__ = get_commit('.') +__version__ = '1.0.0' diff --git a/project.yml b/project.yml index 92ee069..2165475 100644 --- a/project.yml +++ b/project.yml @@ -12,4 +12,19 @@ packages: API_FIGSHARE_TOKEN: ${api_figshare_token} GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} + TOKEN: ${do_token} + - name: trello_reports + functions: + - name: run + runtime: 'python:default' + web: true + limits: + timeout: 420000 + memory: 512 + environment: + API_TRELLO_URL_BASE: ${api_trello_url_base} + API_TRELLO_KEY: ${api_trello_key} + API_TRELLO_TOKEN: ${api_trello_token} + GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} + GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} TOKEN: ${do_token} \ No newline at end of file From 0437f4e7daa9adcf16e0aee9fbf503561d9e3da9 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Fri, 18 Jul 2025 18:01:37 -0400 Subject: [PATCH 03/20] Ignore linter error preventing code before imports (enable local and DO execution using `lib/` folder) --- .flake8 | 3 ++- packages/trello_reports/run/main.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index c06dcf7..c48025e 100644 --- a/.flake8 +++ b/.flake8 @@ -112,7 +112,8 @@ hang-closing = True ignore = E133, E203, - W503 + W503, + E402 # Specify the list of error codes you wish Flake8 to report. select = E, diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py index 40056a5..c706191 100644 --- a/packages/trello_reports/run/main.py +++ b/packages/trello_reports/run/main.py @@ -14,6 +14,7 @@ import functions as f import secrets + def init_argparse(): parser = argparse.ArgumentParser( description='Runs reports for Trello stats' @@ -56,9 +57,9 @@ def run(args): if 'curators' in args.report: print('Running "curators" report') result = f.get_report_date() - #data = curator_report.run(args) - #result = result + f'Running "curator" report completed. {len(data)} curators.' - #if args.sync_to_dashboard: + # data = curator_report.run(args) + # result = result + f'Running "curator" report completed. {len(data)} curators.' + # if args.sync_to_dashboard: # result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' if 'items' in args.report: print('Running "items" report') From 747e04903cca199b3ed535e884f65ebf9ff2c500 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 22 Jul 2025 14:57:12 -0400 Subject: [PATCH 04/20] Add time duration formatting function --- lib/functions.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/lib/functions.py b/lib/functions.py index b0aea7d..511d539 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -1,6 +1,7 @@ from datetime import datetime from os import environ import requests +import re import simplejson as json report_date = None @@ -110,7 +111,77 @@ def format_bytes(bytes, unit, append_unit=False, SI=False): divisor = float(1 << divisors[0]) value = bytes / divisor # return f"{value:,.0f} {unitN}{(value != 1 and len(unitN) > 3)*'s'}" - return f"{value:.2f}" + f' {unit}' if append_unit else f"{value:.2f}" + return f'{value:.2f} {unit}' if append_unit else f'{value:.2f}' + + +def format_duration(duration_string: str, target_unit: str, append_unit: bool=False) -> str: + """ + Converts a duration string from one unit to another. + + Args: + duration_string: A string representing the duration, e.g., "10s", "2m", "3H", "5D", "1W", "0.5M". + Supported units: 's' (seconds), 'm' (minutes), 'H' (hours), + 'D' (days), 'W' (weeks), 'M' (months). + target_unit: The unit to convert to. Supported units are the same as above. + + Returns: + The converted duration as a string. + + Raises: + ValueError: If the input string format is invalid, units are unknown, + or conversion is not possible. + """ + + # Define conversion factors to a base unit (seconds) + # Note: 'M' (months) is approximated as 30 days for simplicity. + # For precise month conversions, a specific date context would be needed. + unit_to_seconds = { + 's': 1, + 'm': 60, + 'H': 60 * 60, + 'D': 24 * 60 * 60, + 'W': 7 * 24 * 60 * 60, + 'M': 30 * 24 * 60 * 60 # Approximation for months + } + + # Validate target unit + if target_unit not in unit_to_seconds: + raise ValueError(f"Unknown target unit: '{target_unit}'. " + "Supported units are 's', 'm', 'H', 'D', 'W', 'M'.") + + # Regular expression to parse the duration string + # It captures a number (integer or float) and then the unit character(s) + match = re.match(r"(\d+(\.\d+)?)([smHDWM])", duration_string) + + if not match: + raise ValueError(f"Invalid duration string format: '{duration_string}'. " + "Expected format: , e.g., '10s', '2.5H'.") + + value_str = match.group(1) + input_unit = match.group(3) + + try: + value = float(value_str) + except ValueError: + # This should ideally be caught by the regex, but as a safeguard + raise ValueError(f"Could not parse numerical value from '{value_str}'.") + + if input_unit not in unit_to_seconds: + raise ValueError(f"Unknown input unit: '{input_unit}'. " + "Supported units are 's', 'm', 'H', 'D', 'W', 'M'.") + + # Convert the input duration to seconds + duration_in_seconds = value * unit_to_seconds[input_unit] + + # Convert from seconds to the target unit + if unit_to_seconds[target_unit] == 0: + # This case should not happen with the current unit_to_seconds, + # but is a good safeguard against division by zero if factors change. + raise ValueError(f"Conversion factor for target unit '{target_unit}' is zero, cannot convert.") + + converted_duration = duration_in_seconds / unit_to_seconds[target_unit] + + return f'{converted_duration:.2f} {target_unit}' if append_unit else f'{converted_duration:.2f}' def get_report_outfile(reportname, prefix=''): From b9780abe1db0757ab3993b67c3d7f845dd9c87a6 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 22 Jul 2025 15:57:46 -0400 Subject: [PATCH 05/20] Add env variables for board ID and list ID for Trello API calls --- lib/secrets.example.py | 2 ++ project.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/secrets.example.py b/lib/secrets.example.py index f4efae2..2ecdc39 100644 --- a/lib/secrets.example.py +++ b/lib/secrets.example.py @@ -12,6 +12,8 @@ api_trello_url_base = '' api_trello_key = '' api_trello_token = '' +trello_board_id = '' +trello_publishedlist_id = '' # Google sheet gsheets_dashboard_post_url = '' diff --git a/project.yml b/project.yml index 2165475..eadd8ae 100644 --- a/project.yml +++ b/project.yml @@ -25,6 +25,8 @@ packages: API_TRELLO_URL_BASE: ${api_trello_url_base} API_TRELLO_KEY: ${api_trello_key} API_TRELLO_TOKEN: ${api_trello_token} + TRELLO_BOARD_ID: ${trello_board_id} + TRELLO_PUBLISHEDLIST_ID: ${trello_publishedlist_id} GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} TOKEN: ${do_token} \ No newline at end of file From fc40d4bf6d10a5fdb2bd59f7ea41110c05a184f5 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 22 Jul 2025 16:11:05 -0400 Subject: [PATCH 06/20] Add stub curator report using py-trello for API calls. Only prints the available custom fields for each card for now --- packages/trello_reports/run/curator_report.py | 79 +++++++++++++++++++ packages/trello_reports/run/main.py | 13 +-- packages/trello_reports/run/requirements.txt | 3 +- 3 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 packages/trello_reports/run/curator_report.py diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py new file mode 100644 index 0000000..e33f4b9 --- /dev/null +++ b/packages/trello_reports/run/curator_report.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +# Prints the usage for each public dataset +# +# Author: Fernando Rios + +import sys +import requests +import simplejson as json +from datetime import datetime +from os import environ +from trello import TrelloClient + +sys.path.insert(0, 'lib/') +import functions as f + + +tc = None + +def run(args): + # Optionally writes a CSV report to file and returns a json array of objects with the data that was written. + + print('Getting curator stats from Trello') + curators = [] + + tc = TrelloClient(api_key=environ['API_TRELLO_KEY'], api_secret=environ['API_TRELLO_TOKEN']) + + try: + board = tc.get_board(environ['TRELLO_BOARD_ID']) + published_list = board.get_list(environ['TRELLO_PUBLISHEDLIST_ID']) + cards = published_list.list_cards() + for card in cards: + print(card.name) + for field in card.custom_fields: + print(f' {field.name}: {field.value}') + except Exception as e: + print(f'Error getting board info: {e}') + + outfile = None + if args.outfile: + outfile = f.get_report_outfile(args.outfile, 'curators') + outfile.write( + f'id,total_items,total_time ({args.units}),' + + f'easy_items,easy_time ({args.units}),med_items,med_time ({args.units}),hard_items,hard_time ({args.units}),' + + f'3M_items,3M_time ({args.units}),6M_items,6M_time ({args.units}),' + + f'1Y_items,1Y_time ({args.units}),2Y_items,2Y_time ({args.units}),' + + f'3M_easy_items,3M_easy_time ({args.units}),6M_easy_items,6M_easy_time ({args.units}),' + + f'1Y_easy_items,1Y_easy_time ({args.units}),2Y_easy_items,2Y_easy_time ({args.units}),' + + f'3M_med_items,3M_med_time ({args.units}),6M_med_items,6M_med_time ({args.units}),' + + f'1Y_med_items,1Y_med_time ({args.units}),2Y_med_items,2Y_med_time ({args.units}),' + + f'3M_hard_items,3M_hard_time ({args.units}),6M_hard_items,6M_hard_time ({args.units})' + + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units})\n') + else: + print(f'id\t,total_items\t,total_time ({args.units})') + + # for article in articles: + # if outfile: + # s = '{0},{1},{2},{3},{4},{11},{5},{6},{7},{8},{9},{10},{12}' + # else: + # s = '{0}\t,{1}\t,{2}\t,{3}\t,{4}\t,{5}' + + # s = s.format(article['id'], article['version'], article['totalfilesize'], + # '"' + article['title'].replace('"', '""') + '"', + # article['type'], + # article['published_date'], article['modified_date'], article['embargo_date'], article['embargo_type'], + # article['embargo_options_type'], article['is_embargoed'], article['is_public'], article['report_date']) + + # if outfile: + # outfile.write(s + '\n') + # elif not args.sync_to_dashboard: + # print(s) + + #print(f'Totals (hours):\t\t{total_usage}') + #print(f'Totals ({args.units}):\t\t{f.format_bytes(total_usage, args.units)}') + + if outfile: + outfile.close() + + return curators diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py index c706191..ae669b4 100644 --- a/packages/trello_reports/run/main.py +++ b/packages/trello_reports/run/main.py @@ -8,9 +8,9 @@ import sys from os import environ from version import __version__, __commit__ -#import items_report sys.path.insert(0, 'lib/') +import curator_report import functions as f import secrets @@ -56,11 +56,10 @@ def run(args): result = '' if 'curators' in args.report: print('Running "curators" report') - result = f.get_report_date() - # data = curator_report.run(args) - # result = result + f'Running "curator" report completed. {len(data)} curators.' - # if args.sync_to_dashboard: - # result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' + data = curator_report.run(args) + result = result + f'Running "curator" report completed. {len(data)} curators.' + if args.sync_to_dashboard: + result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' if 'items' in args.report: print('Running "items" report') result = 'not implemented' @@ -74,6 +73,8 @@ def run(args): environ['API_TRELLO_URL_BASE'] = secrets.api_trello_url_base environ['API_TRELLO_KEY'] = secrets.api_trello_key environ['API_TRELLO_TOKEN'] = secrets.api_trello_token + environ['TRELLO_BOARD_ID'] = secrets.trello_board_id + environ['TRELLO_PUBLISHEDLIST_ID'] = secrets.trello_publishedlist_id environ['GSHEETS_DASHBOARD_POST_URL'] = secrets.gsheets_dashboard_post_url environ['GSHEETS_DASHBOARD_KEY'] = secrets.gsheets_dashboard_key environ['TOKEN'] = secrets.do_token diff --git a/packages/trello_reports/run/requirements.txt b/packages/trello_reports/run/requirements.txt index 4104bca..f5f56a8 100644 --- a/packages/trello_reports/run/requirements.txt +++ b/packages/trello_reports/run/requirements.txt @@ -1,2 +1,3 @@ requests -simplejson \ No newline at end of file +simplejson +py-trello \ No newline at end of file From 76fd5209b09b8a0a49d810105b35ffdb162ae56c Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 22 Jul 2025 23:19:59 -0400 Subject: [PATCH 07/20] Add total items and time per curator to curator report --- packages/trello_reports/run/curator_report.py | 83 +++++++++++++------ 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index e33f4b9..b618a4f 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -5,8 +5,10 @@ # Author: Fernando Rios import sys +import traceback import requests import simplejson as json +from ast import literal_eval from datetime import datetime from os import environ from trello import TrelloClient @@ -21,26 +23,57 @@ def run(args): # Optionally writes a CSV report to file and returns a json array of objects with the data that was written. print('Getting curator stats from Trello') - curators = [] + curators = {} tc = TrelloClient(api_key=environ['API_TRELLO_KEY'], api_secret=environ['API_TRELLO_TOKEN']) try: board = tc.get_board(environ['TRELLO_BOARD_ID']) + boardmembers = board.all_members() published_list = board.get_list(environ['TRELLO_PUBLISHEDLIST_ID']) cards = published_list.list_cards() for card in cards: print(card.name) for field in card.custom_fields: print(f' {field.name}: {field.value}') + if 'reviewer_' in field.name: + if field.value not in curators: + # Add a new curator to the list if they're not already there + # The value of the reviewer_x custom field in trello must be a trello username + curators[field.value] = {'id': '', 'total_items': 0, 'total_time': 0, + 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0,'hard_items': 0, 'hard_time': 0, + '3M_items' :0, '3M_time': 0} + for member in boardmembers: + # Extract the id from the username + if member.username == field.value: + curators[field.value]['id'] = member.id + + curators[field.value]['total_items'] += 1 + + print() + for plugin in card.plugin_data: + if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': + # Get the recorded time from the Activity powerup (convert from milliseconds to seconds) + powerup_timedata = literal_eval(plugin['value'])['users'] + for username, curator in curators.items(): + if curator['id'] in powerup_timedata: + curator['total_time'] += int(powerup_timedata[curator['id']]['time'])/1000 + + print(f' {powerup_timedata}') + except Exception as e: - print(f'Error getting board info: {e}') + tb_list = traceback.extract_tb(sys.exc_info()[2]) + line_number = tb_list[-1][1] + print(f'Error getting board info: {e}, line {line_number}') + return {} + + print('Curation stats for items published since Jan 1, 2025') outfile = None if args.outfile: outfile = f.get_report_outfile(args.outfile, 'curators') outfile.write( - f'id,total_items,total_time ({args.units}),' + f'username,total_items,total_time ({args.units}),' + f'easy_items,easy_time ({args.units}),med_items,med_time ({args.units}),hard_items,hard_time ({args.units}),' + f'3M_items,3M_time ({args.units}),6M_items,6M_time ({args.units}),' + f'1Y_items,1Y_time ({args.units}),2Y_items,2Y_time ({args.units}),' @@ -51,27 +84,29 @@ def run(args): + f'3M_hard_items,3M_hard_time ({args.units}),6M_hard_items,6M_hard_time ({args.units})' + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units})\n') else: - print(f'id\t,total_items\t,total_time ({args.units})') - - # for article in articles: - # if outfile: - # s = '{0},{1},{2},{3},{4},{11},{5},{6},{7},{8},{9},{10},{12}' - # else: - # s = '{0}\t,{1}\t,{2}\t,{3}\t,{4}\t,{5}' - - # s = s.format(article['id'], article['version'], article['totalfilesize'], - # '"' + article['title'].replace('"', '""') + '"', - # article['type'], - # article['published_date'], article['modified_date'], article['embargo_date'], article['embargo_type'], - # article['embargo_options_type'], article['is_embargoed'], article['is_public'], article['report_date']) - - # if outfile: - # outfile.write(s + '\n') - # elif not args.sync_to_dashboard: - # print(s) - - #print(f'Totals (hours):\t\t{total_usage}') - #print(f'Totals ({args.units}):\t\t{f.format_bytes(total_usage, args.units)}') + print(f'username \t total_items \t total_time ({args.units})') + + total_time = 0 + total_items = 0 + + for username, curator in curators.items(): + if outfile: + pass + else: + s = '{0} \t {1} \t\t {2}' + + s = s.format(username, curator['total_items'], f.format_duration(str(curator['total_time'])+'s', args.units)) + + total_time += curator['total_time'] + total_items += curator['total_items'] + + if outfile: + outfile.write(s + '\n') + elif not args.sync_to_dashboard: + print(s) + + print(f'Total curation hours:\t\t\t\t{f.format_duration(str(total_time)+'s', 'H')}') + print(f'Avg items per curator (primary or secondary):\t{total_items / len(curators)}') if outfile: outfile.close() From 7a3254fd6696b6da9c5f62605b890a204a76d271 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Thu, 24 Jul 2025 19:16:06 -0400 Subject: [PATCH 08/20] Generate all fields for curator report --- lib/functions.py | 17 +- packages/trello_reports/run/curator_report.py | 368 +++++++++++++++--- packages/trello_reports/run/main.py | 2 +- 3 files changed, 337 insertions(+), 50 deletions(-) diff --git a/lib/functions.py b/lib/functions.py index 511d539..562b5cf 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import date, datetime from os import environ import requests import re @@ -26,6 +26,21 @@ def get_report_date(): return report_date +def get_cardlist_filter(): + """ + Returns a dict containing a filter to be used with board.list_cards(), along with a description. + Refer to https://developer.atlassian.com/cloud/trello/guides/rest-api/nested-resources/#cards-nested-resource + for available filters. + """ + + # Limit to this for now since that's as far back as old cards have been updated + since = date(2025, 1, 1) + + # To limit the number of results for testing, add limit parameter to the query. E.g., 'limit': '5' + return {'query': {'since': since.strftime('%a %d %b %Y')}, + 'description': f'Since {since.strftime('%a %d %b %Y')}'} + + def sync_to_dashboard(data, report): """ Sends the given data to the Google Sheet given by report via the sheet's POST request url diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index b618a4f..ad2f8a7 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -6,10 +6,11 @@ import sys import traceback -import requests -import simplejson as json from ast import literal_eval -from datetime import datetime +from datetime import date, datetime +from dateutil.relativedelta import relativedelta +from dateutil.parser import parse +from dateutil.tz import tzlocal from os import environ from trello import TrelloClient @@ -18,56 +19,292 @@ tc = None +currdate = datetime.now(tzlocal()) + + + + +def get_card_curators(card, existing_curators: dict[str, any]=None): + """ + Returns the curators for the given Card object as a list of Trello usernames. + + If the existing_curators dictionary is passed in, if a curator is found that does not exist in the + dictionary, a new entry is created with the key equal to the value of the reviewer_N custom field (N is an integer). + Currently it is the value of the Trello username. + """ + + curators = [] + global tc + + print(card.name) + for field in card.custom_fields: + print(f' {field.name}: {field.value}') + if 'reviewer_' in field.name: + + curators.append(field.value) + + if type(existing_curators) is dict and field.value not in existing_curators: + # Add a new curator to the list if they're not already there + # The value of the reviewer_x custom field in trello must be a trello username + existing_curators[field.value] = {'id': '', 'total_items': 0, 'total_time': 0, + 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0,'hard_items': 0, 'hard_time': 0, + '3M_items' :0, '3M_time': 0, + '6M_items' :0, '6M_time': 0, + '1Y_items' :0, '1Y_time': 0, + '2Y_items' :0, '2Y_time': 0, + '3M_easy_items': 0, '3M_easy_time': 0, + '3M_med_items': 0, '3M_med_time': 0, + '3M_hard_items': 0, '3M_hard_time': 0, + '6M_easy_items': 0, '6M_easy_time': 0, + '6M_med_items': 0, '6M_med_time': 0, + '6M_hard_items': 0, '6M_hard_time': 0, + '1Y_easy_items': 0, '1Y_easy_time': 0, + '1Y_med_items': 0, '1Y_med_time': 0, + '1Y_hard_items': 0, '1Y_hard_time': 0, + '2Y_easy_items': 0, '2Y_easy_time': 0, + '2Y_med_items': 0, '2Y_med_time': 0, + '2Y_hard_items': 0, '2Y_hard_time': 0} + + for member in tc.get_board(card.board_id).all_members(): + # Extract the id from the username + if member.username == field.value: + existing_curators[field.value]['id'] = member.id + + # uncomment for debugging + for plugin in card.plugin_data: + if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': + powerup_timedata = literal_eval(plugin['value'])['users'] + print(f' timedata: {powerup_timedata}') + + return curators + + +def get_card_time(card, userid: str=None): + """ + Returns the total time in seconds from the Activity powerup for this card. + If userid is given, limits the time to only the given userid + """ + + total = 0 + + for plugin in card.plugin_data: + if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': + powerup_timedata = literal_eval(plugin['value'])['users'] + for id, timedata in powerup_timedata.items(): + if userid and userid == id: + total = int(timedata['time']) / 1000 + if not userid: + total += int(timedata['time']) / 1000 + return total + + +def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: str, card, cardfilter: dict[str, str]=None): + """ + For the given curators dictionary as generated by get_card_curators + + - Increments the [curators_field_prefix]_items count for each curator that contributed to the given card. + - Increments the [curators_field_prefix]_time value (in seconds) for each curator that contributed to the given card. + The increment is based on the logged time for that curator. + - Applies the operations only if the card meets ALL the criteria specified in cardfilter. If None, the operation is always applied. + + existing_curators is a dict set up by get_card_curators + """ + + # Turn the custom fields into a dict for ease of use + customfields = {} + for field in card.custom_fields: + customfields[field.name] = field.value + + include_card = False + if not cardfilter: + include_card = True + else: + filter_result = {} + for name, value in cardfilter.items(): + filter_result[name] = False + match name: + case 'difficulty': + if name in customfields.keys() and value == customfields[name]: + filter_result[name] = True + + # uncomment for debugging + print(f' {name} filter: result={filter_result[name]} | requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else ''}') + case 'published_date': + pubdate = '' + cutoffdate = '' + if name in customfields.keys(): + pubdate = parse(customfields[name]) + cutoffdate = currdate - relativedelta(months=float(value)) + if pubdate and pubdate >= cutoffdate: + filter_result[name] = True + + # uncomment for debugging + print(f' {name} filter: result={filter_result[name]} | card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') + include_card = all(filter_result.values()) + + # uncomment for debugging + print(f' include_card: {include_card} | {filter_result}') + print(' ---') + + if include_card: + for username, curator in existing_curators.items(): + # Item counts for each curator, based on card filter criteria + for fieldname, fieldvalue in customfields.items(): + if 'reviewer_' in fieldname and username in fieldvalue: + curator[curators_field_prefix + '_items'] += 1 + + # Time for each curator, based on card filter criteria + curator[curators_field_prefix + '_time'] += get_card_time(card, curator['id']) + + +def curator_total_items_time(existing_curators: dict[str, any], card): + return curator_items_time(existing_curators, 'total', card, None) + + +def curator_easy_items_time(existing_curators: dict[str, any], card): + # All items with difficulty=easy + return curator_items_time(existing_curators, 'easy', card, {'difficulty': 'easy'}) + + +def curator_med_items_time(existing_curators: dict[str, any], card): + # All items with difficulty=medium + return curator_items_time(existing_curators, 'med', card, {'difficulty': 'medium'}) + + +def curator_hard_items_time(existing_curators: dict[str, any], card): + # All items with difficulty=hard + return curator_items_time(existing_curators, 'hard', card, {'difficulty': 'hard'}) + + +def curator_3m_items_time(existing_curators: dict[str, any], card): + # All items published in last 3 months + return curator_items_time(existing_curators, '3M', card, {'published_date': 3}) + + +def curator_6m_items_time(existing_curators: dict[str, any], card): + # All items published in last 6 months + return curator_items_time(existing_curators, '6M', card, {'published_date': 6}) + + +def curator_1y_items_time(existing_curators: dict[str, any], card): + # All items published in last year + return curator_items_time(existing_curators, '1Y', card, {'published_date': 12}) + + +def curator_2y_items_time(existing_curators: dict[str, any], card): + # All items published in last 2 years + return curator_items_time(existing_curators, '2Y', card, {'published_date': 24}) + + +def curator_3m_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last 3 months + return curator_items_time(existing_curators, '3M_easy', card, {'difficulty': 'easy', 'published_date': 3}) + + +def curator_6m_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last 6 months + return curator_items_time(existing_curators, '6M_easy', card, {'difficulty': 'easy', 'published_date': 6}) + + +def curator_1y_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last year + return curator_items_time(existing_curators, '1Y_easy', card, {'difficulty': 'easy', 'published_date': 12}) + + +def curator_2y_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last 2 years + return curator_items_time(existing_curators, '2Y_easy', card, {'difficulty': 'easy', 'published_date': 24}) + + +def curator_3m_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 3 months + return curator_items_time(existing_curators, '3M_med', card, {'difficulty': 'medium', 'published_date': 3}) + + +def curator_6m_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 6 months + return curator_items_time(existing_curators, '6M_med', card, {'difficulty': 'medium', 'published_date': 6}) + + +def curator_1y_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last year + return curator_items_time(existing_curators, '1Y_med', card, {'difficulty': 'medium', 'published_date': 12}) + + +def curator_2y_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 2 years + return curator_items_time(existing_curators, '2Y_med', card, {'difficulty': 'medium', 'published_date': 24}) + + +def curator_3m_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 3 months + return curator_items_time(existing_curators, '3M_hard', card, {'difficulty': 'hard', 'published_date': 3}) + + +def curator_6m_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 6 months + return curator_items_time(existing_curators, '6M_hard', card, {'difficulty': 'hard', 'published_date': 6}) + + +def curator_1y_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last year + return curator_items_time(existing_curators, '1Y_hard', card, {'difficulty': 'hard', 'published_date': 12}) + + +def curator_2y_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 2 years + return curator_items_time(existing_curators, '2Y_hard', card, {'difficulty': 'hard', 'published_date': 24}) + def run(args): - # Optionally writes a CSV report to file and returns a json array of objects with the data that was written. + # Optionally writes a CSV report to file and returns a JSON array of objects with the data that was written. - print('Getting curator stats from Trello') + global tc curators = {} + print('Getting curator stats from Trello') + tc = TrelloClient(api_key=environ['API_TRELLO_KEY'], api_secret=environ['API_TRELLO_TOKEN']) - + try: board = tc.get_board(environ['TRELLO_BOARD_ID']) - boardmembers = board.all_members() published_list = board.get_list(environ['TRELLO_PUBLISHEDLIST_ID']) - cards = published_list.list_cards() + cards = published_list.list_cards(query=f.get_cardlist_filter()['query']) + for card in cards: - print(card.name) - for field in card.custom_fields: - print(f' {field.name}: {field.value}') - if 'reviewer_' in field.name: - if field.value not in curators: - # Add a new curator to the list if they're not already there - # The value of the reviewer_x custom field in trello must be a trello username - curators[field.value] = {'id': '', 'total_items': 0, 'total_time': 0, - 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0,'hard_items': 0, 'hard_time': 0, - '3M_items' :0, '3M_time': 0} - for member in boardmembers: - # Extract the id from the username - if member.username == field.value: - curators[field.value]['id'] = member.id - - curators[field.value]['total_items'] += 1 - + # Preprocess the card to get the reviewers + get_card_curators(card, curators) + + curator_total_items_time(curators, card) + curator_easy_items_time(curators, card) + curator_med_items_time(curators, card) + curator_hard_items_time(curators, card) + curator_3m_items_time(curators, card) + curator_6m_items_time(curators, card) + curator_1y_items_time(curators, card) + curator_2y_items_time(curators, card) + curator_3m_easy_items_time(curators, card) + curator_6m_easy_items_time(curators, card) + curator_1y_easy_items_time(curators, card) + curator_2y_easy_items_time(curators, card) + curator_3m_med_items_time(curators, card) + curator_6m_med_items_time(curators, card) + curator_1y_med_items_time(curators, card) + curator_2y_med_items_time(curators, card) + curator_3m_hard_items_time(curators, card) + curator_6m_hard_items_time(curators, card) + curator_1y_hard_items_time(curators, card) + curator_2y_hard_items_time(curators, card) + print() - for plugin in card.plugin_data: - if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': - # Get the recorded time from the Activity powerup (convert from milliseconds to seconds) - powerup_timedata = literal_eval(plugin['value'])['users'] - for username, curator in curators.items(): - if curator['id'] in powerup_timedata: - curator['total_time'] += int(powerup_timedata[curator['id']]['time'])/1000 - - print(f' {powerup_timedata}') - + except Exception as e: tb_list = traceback.extract_tb(sys.exc_info()[2]) line_number = tb_list[-1][1] print(f'Error getting board info: {e}, line {line_number}') return {} - print('Curation stats for items published since Jan 1, 2025') + print(f'Curation stats for items (and versions) published since {f.get_cardlist_filter()['description']}') outfile = None if args.outfile: @@ -77,25 +314,59 @@ def run(args): + f'easy_items,easy_time ({args.units}),med_items,med_time ({args.units}),hard_items,hard_time ({args.units}),' + f'3M_items,3M_time ({args.units}),6M_items,6M_time ({args.units}),' + f'1Y_items,1Y_time ({args.units}),2Y_items,2Y_time ({args.units}),' - + f'3M_easy_items,3M_easy_time ({args.units}),6M_easy_items,6M_easy_time ({args.units}),' - + f'1Y_easy_items,1Y_easy_time ({args.units}),2Y_easy_items,2Y_easy_time ({args.units}),' - + f'3M_med_items,3M_med_time ({args.units}),6M_med_items,6M_med_time ({args.units}),' - + f'1Y_med_items,1Y_med_time ({args.units}),2Y_med_items,2Y_med_time ({args.units}),' - + f'3M_hard_items,3M_hard_time ({args.units}),6M_hard_items,6M_hard_time ({args.units})' - + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units})\n') + + f'3M_easy_items,3M_easy_time ({args.units}),3M_med_items,3M_med_time ({args.units}),' + + f'3M_hard_items,3M_hard_time ({args.units}),6M_easy_items,6M_easy_time ({args.units}),' + + f'6M_med_items,6M_med_time ({args.units}),6M_hard_items,6M_hard_time ({args.units}),' + + f'1Y_easy_items,1Y_easy_time ({args.units}),1Y_med_items,1Y_med_time ({args.units}),' + + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_easy_items,2Y_easy_time ({args.units}),' + + f'2Y_med_items,2Y_med_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units})\n') + else: - print(f'username \t total_items \t total_time ({args.units})') + print(f'username\ttotal_items\ttotal_time ({args.units})\tlast 3m items\tlast 3m time ({args.units})') total_time = 0 total_items = 0 for username, curator in curators.items(): if outfile: - pass + s = ('{0},{1},{2},' # username, total items, total time + + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + + '{9},{10},{11},{12},' # 3M, 6M items & time + + '{13},{14},{15},{16},' # 1Y, 2Y items & time + + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time + + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time + + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time + + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time + + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time + + '{37},{38},{39},{40}') # 2Y_med, 2Y_hard items & time else: - s = '{0} \t {1} \t\t {2}' - - s = s.format(username, curator['total_items'], f.format_duration(str(curator['total_time'])+'s', args.units)) + s = ('{0} \t' + + '{1} \t\t {2} \t\t' + + '{9} \t\t {10} \t') + + s = s.format( + username, # 0 + curator['total_items'], f.format_duration(str(curator['total_time'])+'s', args.units), # 1,2 + curator['easy_items'], f.format_duration(str(curator['easy_time'])+'s', args.units), # 3,4 + curator['med_items'], f.format_duration(str(curator['med_time'])+'s', args.units), # 5,6 + curator['hard_items'], f.format_duration(str(curator['hard_time'])+'s', args.units), # 7,8 + curator['3M_items'], f.format_duration(str(curator['3M_time'])+'s', args.units), # 9,10 + curator['6M_items'], f.format_duration(str(curator['6M_time'])+'s', args.units), # 11,12 + curator['1Y_items'], f.format_duration(str(curator['1Y_time'])+'s', args.units), # 13,14 + curator['2Y_items'], f.format_duration(str(curator['2Y_time'])+'s', args.units), # 15,16 + curator['3M_easy_items'], f.format_duration(str(curator['3M_easy_time'])+'s', args.units), # 17,18 + curator['3M_med_items'], f.format_duration(str(curator['3M_med_time'])+'s', args.units), # 19,20 + curator['3M_hard_items'], f.format_duration(str(curator['3M_hard_time'])+'s', args.units), # 21,22 + curator['6M_easy_items'], f.format_duration(str(curator['6M_easy_time'])+'s', args.units), # 23,24 + curator['6M_med_items'], f.format_duration(str(curator['6M_med_time'])+'s', args.units), # 25,26 + curator['6M_hard_items'], f.format_duration(str(curator['6M_hard_time'])+'s', args.units), # 27,28 + curator['1Y_easy_items'], f.format_duration(str(curator['1Y_easy_time'])+'s', args.units), # 29,30 + curator['1Y_med_items'], f.format_duration(str(curator['1Y_med_time'])+'s', args.units), # 31,32 + curator['1Y_hard_items'], f.format_duration(str(curator['1Y_hard_time'])+'s', args.units), # 33,34 + curator['2Y_easy_items'], f.format_duration(str(curator['2Y_easy_time'])+'s', args.units), # 35,36 + curator['2Y_med_items'], f.format_duration(str(curator['2Y_med_time'])+'s', args.units), # 37,38 + curator['2Y_hard_items'], f.format_duration(str(curator['2Y_hard_time'])+'s', args.units)) # 39,40 + total_time += curator['total_time'] total_items += curator['total_items'] @@ -105,8 +376,9 @@ def run(args): elif not args.sync_to_dashboard: print(s) + print() print(f'Total curation hours:\t\t\t\t{f.format_duration(str(total_time)+'s', 'H')}') - print(f'Avg items per curator (primary or secondary):\t{total_items / len(curators)}') + print(f'Avg hours per item:\t{f.format_duration(str(total_time/total_items)+'s', 'H')}') if outfile: outfile.close() diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py index ae669b4..ab9609a 100644 --- a/packages/trello_reports/run/main.py +++ b/packages/trello_reports/run/main.py @@ -59,7 +59,7 @@ def run(args): data = curator_report.run(args) result = result + f'Running "curator" report completed. {len(data)} curators.' if args.sync_to_dashboard: - result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' + result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' if 'items' in args.report: print('Running "items" report') result = 'not implemented' From 01fafe3c368ab686ea4ebc9e265c3ffdeec73747 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Fri, 25 Jul 2025 14:44:25 -0400 Subject: [PATCH 09/20] Remove the board query filter and list filter and add some debugging code The filter was excluding some cards that should not have been excluded --- lib/functions.py | 7 +--- lib/secrets.example.py | 1 - packages/trello_reports/run/curator_report.py | 37 +++++++++++++++---- project.yml | 1 - 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/functions.py b/lib/functions.py index 562b5cf..6d5fc70 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -33,12 +33,9 @@ def get_cardlist_filter(): for available filters. """ - # Limit to this for now since that's as far back as old cards have been updated - since = date(2025, 1, 1) - # To limit the number of results for testing, add limit parameter to the query. E.g., 'limit': '5' - return {'query': {'since': since.strftime('%a %d %b %Y')}, - 'description': f'Since {since.strftime('%a %d %b %Y')}'} + return {'query': {'customFieldItems': 'true'}, + 'description': f'since the beginning'} def sync_to_dashboard(data, report): diff --git a/lib/secrets.example.py b/lib/secrets.example.py index 2ecdc39..4d391d0 100644 --- a/lib/secrets.example.py +++ b/lib/secrets.example.py @@ -13,7 +13,6 @@ api_trello_key = '' api_trello_token = '' trello_board_id = '' -trello_publishedlist_id = '' # Google sheet gsheets_dashboard_post_url = '' diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index ad2f8a7..0afb13c 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -116,6 +116,7 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: customfields[field.name] = field.value include_card = False + filter_result = 'NoFilter' if not cardfilter: include_card = True else: @@ -142,9 +143,9 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: print(f' {name} filter: result={filter_result[name]} | card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') include_card = all(filter_result.values()) - # uncomment for debugging - print(f' include_card: {include_card} | {filter_result}') - print(' ---') + # uncomment for debugging + print(f' include_card: {include_card} | {filter_result}') + print(' ---') if include_card: for username, curator in existing_curators.items(): @@ -152,6 +153,7 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: for fieldname, fieldvalue in customfields.items(): if 'reviewer_' in fieldname and username in fieldvalue: curator[curators_field_prefix + '_items'] += 1 + if 'reviewer_1' in fieldname: curator[curators_field_prefix + '_reviewer1_items'] += 1 # Time for each curator, based on card filter criteria curator[curators_field_prefix + '_time'] += get_card_time(card, curator['id']) @@ -268,10 +270,30 @@ def run(args): try: board = tc.get_board(environ['TRELLO_BOARD_ID']) - published_list = board.get_list(environ['TRELLO_PUBLISHEDLIST_ID']) - cards = published_list.list_cards(query=f.get_cardlist_filter()['query']) + cards = board.get_cards(filters=f.get_cardlist_filter()['query']) + # Preprocess board cards to filter out the ones without published_date set. + # Do it this way instead of by returning only cards from the Published list so that + # we grab cards in other lists that may have been published already but have been moved to another list. + publishedcards = [] + fl = None + + # Uncomment to write the cards with published_date set to file for debugging purposes + # fl = open('publishedcards.csv', 'w', encoding="utf-8") + + if fl: fl.write('name,published_date\n') for card in cards: + keep = False + for field in card.custom_fields: + if field.name == 'published_date' and not field.value == '': + keep = True + break + if keep: + publishedcards.append(card) + if fl: fl.write(f'"{card.name.replace('"','""')}","{field.value}"\n') + if fl: fl.close() + + for card in publishedcards: # Preprocess the card to get the reviewers get_card_curators(card, curators) @@ -304,7 +326,8 @@ def run(args): print(f'Error getting board info: {e}, line {line_number}') return {} - print(f'Curation stats for items (and versions) published since {f.get_cardlist_filter()['description']}') + print(f'Curation stats for items (and versions) published {f.get_cardlist_filter()['description']}') + print(f'Processed {len(publishedcards)} cards with published_date set, out of {len(cards)} fetched') outfile = None if args.outfile: @@ -377,7 +400,7 @@ def run(args): print(s) print() - print(f'Total curation hours:\t\t\t\t{f.format_duration(str(total_time)+'s', 'H')}') + print(f'Total curation hours:\t{f.format_duration(str(total_time)+'s', 'H')}') print(f'Avg hours per item:\t{f.format_duration(str(total_time/total_items)+'s', 'H')}') if outfile: diff --git a/project.yml b/project.yml index eadd8ae..276e705 100644 --- a/project.yml +++ b/project.yml @@ -26,7 +26,6 @@ packages: API_TRELLO_KEY: ${api_trello_key} API_TRELLO_TOKEN: ${api_trello_token} TRELLO_BOARD_ID: ${trello_board_id} - TRELLO_PUBLISHEDLIST_ID: ${trello_publishedlist_id} GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} TOKEN: ${do_token} \ No newline at end of file From b8163c950e348645b0ae37b70b26316cd6b07ed5 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Mon, 28 Jul 2025 15:39:44 -0400 Subject: [PATCH 10/20] lint --- packages/trello_reports/run/curator_report.py | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index 0afb13c..3faac39 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -7,7 +7,7 @@ import sys import traceback from ast import literal_eval -from datetime import date, datetime +from datetime import datetime from dateutil.relativedelta import relativedelta from dateutil.parser import parse from dateutil.tz import tzlocal @@ -22,19 +22,16 @@ currdate = datetime.now(tzlocal()) - - -def get_card_curators(card, existing_curators: dict[str, any]=None): +def get_card_curators(card, existing_curators: dict[str, any] = None): """ Returns the curators for the given Card object as a list of Trello usernames. - - If the existing_curators dictionary is passed in, if a curator is found that does not exist in the + + If the existing_curators dictionary is passed in, if a curator is found that does not exist in the dictionary, a new entry is created with the key equal to the value of the reviewer_N custom field (N is an integer). Currently it is the value of the Trello username. """ - + curators = [] - global tc print(card.name) for field in card.custom_fields: @@ -46,25 +43,43 @@ def get_card_curators(card, existing_curators: dict[str, any]=None): if type(existing_curators) is dict and field.value not in existing_curators: # Add a new curator to the list if they're not already there # The value of the reviewer_x custom field in trello must be a trello username - existing_curators[field.value] = {'id': '', 'total_items': 0, 'total_time': 0, - 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0,'hard_items': 0, 'hard_time': 0, - '3M_items' :0, '3M_time': 0, - '6M_items' :0, '6M_time': 0, - '1Y_items' :0, '1Y_time': 0, - '2Y_items' :0, '2Y_time': 0, - '3M_easy_items': 0, '3M_easy_time': 0, - '3M_med_items': 0, '3M_med_time': 0, - '3M_hard_items': 0, '3M_hard_time': 0, - '6M_easy_items': 0, '6M_easy_time': 0, - '6M_med_items': 0, '6M_med_time': 0, - '6M_hard_items': 0, '6M_hard_time': 0, - '1Y_easy_items': 0, '1Y_easy_time': 0, - '1Y_med_items': 0, '1Y_med_time': 0, - '1Y_hard_items': 0, '1Y_hard_time': 0, - '2Y_easy_items': 0, '2Y_easy_time': 0, - '2Y_med_items': 0, '2Y_med_time': 0, - '2Y_hard_items': 0, '2Y_hard_time': 0} - + existing_curators[field.value] = { + 'id': '', 'total_items': 0, 'total_time': 0, + 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0,'hard_items': 0, 'hard_time': 0, + '3M_items': 0, '3M_time': 0, + '6M_items': 0, '6M_time': 0, + '1Y_items': 0, '1Y_time': 0, + '2Y_items' :0, '2Y_time': 0, + '3M_easy_items': 0, '3M_easy_time': 0, + '3M_med_items': 0, '3M_med_time': 0, + '3M_hard_items': 0, '3M_hard_time': 0, + '6M_easy_items': 0, '6M_easy_time': 0, + '6M_med_items': 0, '6M_med_time': 0, + '6M_hard_items': 0, '6M_hard_time': 0, + '1Y_easy_items': 0, '1Y_easy_time': 0, + '1Y_med_items': 0, '1Y_med_time': 0, + '1Y_hard_items': 0, '1Y_hard_time': 0, + '2Y_easy_items': 0, '2Y_easy_time': 0, + '2Y_med_items': 0, '2Y_med_time': 0, + '2Y_hard_items': 0, '2Y_hard_time': 0, + 'total_reviewer1_items': 0, 'total_reviewer1_time': 0, + 'total_reviewer2_items': 0, 'total_reviewer2_time': 0, + 'easy_reviewer1_items': 0, 'easy_reviewer1_time': 0, + 'easy_reviewer2_items': 0, 'easy_reviewer2_time': 0, + 'med_reviewer1_items': 0, 'med_reviewer1_time': 0, + 'med_reviewer2_items': 0, 'med_reviewer2_time': 0, + 'hard_reviewer1_items': 0, 'hard_reviewer1_time': 0, + 'hard_reviewer2_items': 0, 'hard_reviewer2_time': 0, + '3M_reviewer1_items': 0, '3M_reviewer1_time': 0, + '3M_reviewer2_items': 0, '3M_reviewer2_time': 0, + '6M_reviewer1_items': 0, '6M_reviewer1_time': 0, + '6M_reviewer2_items': 0, '6M_reviewer2_time': 0, + '1Y_reviewer1_items': 0, '1Y_reviewer1_time': 0, + '1Y_reviewer2_items': 0, '1Y_reviewer2_time': 0, + '2Y_reviewer1_items': 0, '2Y_reviewer1_time': 0, + '2Y_reviewer2_items': 0, '2Y_reviewer2_time': 0 + } + for member in tc.get_board(card.board_id).all_members(): # Extract the id from the username if member.username == field.value: @@ -277,7 +292,7 @@ def run(args): # we grab cards in other lists that may have been published already but have been moved to another list. publishedcards = [] fl = None - + # Uncomment to write the cards with published_date set to file for debugging purposes # fl = open('publishedcards.csv', 'w', encoding="utf-8") @@ -292,7 +307,7 @@ def run(args): publishedcards.append(card) if fl: fl.write(f'"{card.name.replace('"','""')}","{field.value}"\n') if fl: fl.close() - + for card in publishedcards: # Preprocess the card to get the reviewers get_card_curators(card, curators) From 8432d637890f1f677d52e1af240942ace4cc6426 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 10:34:24 -0400 Subject: [PATCH 11/20] add more curator fields to report and extra debugging code Debugging code can be turned off with the `debug` variable --- lib/functions.py | 5 +- packages/trello_reports/run/curator_report.py | 222 +++++++++++++++--- packages/trello_reports/run/main.py | 1 - 3 files changed, 191 insertions(+), 37 deletions(-) diff --git a/lib/functions.py b/lib/functions.py index 6d5fc70..c18e6a6 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -34,8 +34,9 @@ def get_cardlist_filter(): """ # To limit the number of results for testing, add limit parameter to the query. E.g., 'limit': '5' - return {'query': {'customFieldItems': 'true'}, - 'description': f'since the beginning'} + # 1000 is the maximum limit (don't change unless for testing) + return {'query': {'customFieldItems': 'true', 'sort': '-id', 'limit': '1000'}, + 'description': f'fetched cards with published_date set.'} def sync_to_dashboard(data, report): diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index 3faac39..2e1762f 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -17,7 +17,7 @@ sys.path.insert(0, 'lib/') import functions as f - +debug = 0 # 0 = off, 1 = print card name + output cards to cards.csv, 2 = 1 + print more card info, 3 = 2 + card filter results tc = None currdate = datetime.now(tzlocal()) @@ -33,11 +33,11 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): curators = [] - print(card.name) + if debug > 0: print(card.name) + for field in card.custom_fields: - print(f' {field.name}: {field.value}') if 'reviewer_' in field.name: - + if debug > 1: print(f' {field.name}: {field.value}') curators.append(field.value) if type(existing_curators) is dict and field.value not in existing_curators: @@ -77,7 +77,31 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): '1Y_reviewer1_items': 0, '1Y_reviewer1_time': 0, '1Y_reviewer2_items': 0, '1Y_reviewer2_time': 0, '2Y_reviewer1_items': 0, '2Y_reviewer1_time': 0, - '2Y_reviewer2_items': 0, '2Y_reviewer2_time': 0 + '2Y_reviewer2_items': 0, '2Y_reviewer2_time': 0, + '3M_easy_reviewer1_items': 0, '3M_easy_reviewer1_time': 0, + '3M_easy_reviewer2_items': 0, '3M_easy_reviewer2_time': 0, + '6M_easy_reviewer1_items': 0, '6M_easy_reviewer1_time': 0, + '6M_easy_reviewer2_items': 0, '6M_easy_reviewer2_time': 0, + '1Y_easy_reviewer1_items': 0, '1Y_easy_reviewer1_time': 0, + '1Y_easy_reviewer2_items': 0, '1Y_easy_reviewer2_time': 0, + '2Y_easy_reviewer1_items': 0, '2Y_easy_reviewer1_time': 0, + '2Y_easy_reviewer2_items': 0, '2Y_easy_reviewer2_time': 0, + '3M_med_reviewer1_items': 0, '3M_med_reviewer1_time': 0, + '3M_med_reviewer2_items': 0, '3M_med_reviewer2_time': 0, + '6M_med_reviewer1_items': 0, '6M_med_reviewer1_time': 0, + '6M_med_reviewer2_items': 0, '6M_med_reviewer2_time': 0, + '1Y_med_reviewer1_items': 0, '1Y_med_reviewer1_time': 0, + '1Y_med_reviewer2_items': 0, '1Y_med_reviewer2_time': 0, + '2Y_med_reviewer1_items': 0, '2Y_med_reviewer1_time': 0, + '2Y_med_reviewer2_items': 0, '2Y_med_reviewer2_time': 0, + '3M_hard_reviewer1_items': 0, '3M_hard_reviewer1_time': 0, + '3M_hard_reviewer2_items': 0, '3M_hard_reviewer2_time': 0, + '6M_hard_reviewer1_items': 0, '6M_hard_reviewer1_time': 0, + '6M_hard_reviewer2_items': 0, '6M_hard_reviewer2_time': 0, + '1Y_hard_reviewer1_items': 0, '1Y_hard_reviewer1_time': 0, + '1Y_hard_reviewer2_items': 0, '1Y_hard_reviewer2_time': 0, + '2Y_hard_reviewer1_items': 0, '2Y_hard_reviewer1_time': 0, + '2Y_hard_reviewer2_items': 0, '2Y_hard_reviewer2_time': 0 } for member in tc.get_board(card.board_id).all_members(): @@ -85,8 +109,8 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): if member.username == field.value: existing_curators[field.value]['id'] = member.id - # uncomment for debugging - for plugin in card.plugin_data: + if debug > 1: + for plugin in card.plugin_data: if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': powerup_timedata = literal_eval(plugin['value'])['users'] print(f' timedata: {powerup_timedata}') @@ -125,6 +149,8 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: existing_curators is a dict set up by get_card_curators """ + if debug > 2: print(f'cardfilter: {cardfilter}') + # Turn the custom fields into a dict for ease of use customfields = {} for field in card.custom_fields: @@ -143,8 +169,9 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: if name in customfields.keys() and value == customfields[name]: filter_result[name] = True - # uncomment for debugging - print(f' {name} filter: result={filter_result[name]} | requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else ''}') + if debug > 2: + print(f' {name} filter: result={filter_result[name]} | ' + + f'requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else ''}') case 'published_date': pubdate = '' cutoffdate = '' @@ -154,25 +181,49 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: if pubdate and pubdate >= cutoffdate: filter_result[name] = True - # uncomment for debugging - print(f' {name} filter: result={filter_result[name]} | card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') + if debug > 2: + print(f' {name} filter: result={filter_result[name]} | ' + + f'card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') + case 'reviewer_1' | 'reviewer_2': + if name in customfields.keys(): + filter_result[name] = True + include_card = all(filter_result.values()) - # uncomment for debugging - print(f' include_card: {include_card} | {filter_result}') - print(' ---') + if debug > 2: + print(f' include_card: {include_card} | {filter_result}') + print(' ---') if include_card: for username, curator in existing_curators.items(): + reviewer1_id = '' + reviewer2_id = '' + # Item counts for each curator, based on card filter criteria for fieldname, fieldvalue in customfields.items(): if 'reviewer_' in fieldname and username in fieldvalue: + + # Add to the item count, whether the curator was reviewer 1 or reviewer 2 curator[curators_field_prefix + '_items'] += 1 - if 'reviewer_1' in fieldname: curator[curators_field_prefix + '_reviewer1_items'] += 1 + + # Now, add to the specific reviewer 1 or 2 counts for the given curator + # and record whether that curator was reviewer 1 or 2. + if 'reviewer_1' in fieldname and username in fieldvalue: + curator[curators_field_prefix + '_reviewer1_items'] += 1 + reviewer1_id = curator['id'] + if 'reviewer_2' in fieldname and username in fieldvalue: + curator[curators_field_prefix + '_reviewer2_items'] += 1 + reviewer2_id = curator['id'] # Time for each curator, based on card filter criteria curator[curators_field_prefix + '_time'] += get_card_time(card, curator['id']) + # Record time for the appropriate reviewer1 or reviewer2 categories for this curator (if this curator worked on the item) + if reviewer1_id: + curator[curators_field_prefix + '_reviewer1_time'] += get_card_time(card, reviewer1_id) + if reviewer2_id: + curator[curators_field_prefix + '_reviewer2_time'] += get_card_time(card, reviewer2_id) + def curator_total_items_time(existing_curators: dict[str, any], card): return curator_items_time(existing_curators, 'total', card, None) @@ -293,19 +344,30 @@ def run(args): publishedcards = [] fl = None - # Uncomment to write the cards with published_date set to file for debugging purposes - # fl = open('publishedcards.csv', 'w', encoding="utf-8") - - if fl: fl.write('name,published_date\n') + if debug > 0: fl = open('cards.csv', 'w', encoding="utf-8") + + if fl: fl.write( + 'name,difficulty,total_time_mins,article_version,article_revision,' + + 'reviewer_1,reviewer_2,submission_date,forms_sent_date,reviewed_date,published_date,keep?\n') for card in cards: - keep = False + # Turn the custom fields into a dict for ease of use + customfields = {} for field in card.custom_fields: - if field.name == 'published_date' and not field.value == '': - keep = True - break + customfields[field.name] = field.value + keep = True if not customfields.get('published_date', '') == '' else False + if keep: publishedcards.append(card) - if fl: fl.write(f'"{card.name.replace('"','""')}","{field.value}"\n') + + if fl: fl.write( + f'"{card.name.replace('"','""')}",' + + f'{customfields.get('difficulty', '""')},' + + f'{customfields.get('total_time_mins', '""')},' + + f'{customfields.get('article_version', '""')},{customfields.get('article_revision', '""')},' + + f'{customfields.get('reviewer_1', '""')},{customfields.get('reviewer_2', '""')},' + + f'{customfields.get('submission_date', '""')},{customfields.get('forms_sent_date', '""')},' + + f'{customfields.get('reviewed_date', '""')},{customfields.get('published_date', '""')},' + + f'{keep}\n') if fl: fl.close() for card in publishedcards: @@ -332,13 +394,13 @@ def run(args): curator_6m_hard_items_time(curators, card) curator_1y_hard_items_time(curators, card) curator_2y_hard_items_time(curators, card) + print() - print() except Exception as e: tb_list = traceback.extract_tb(sys.exc_info()[2]) line_number = tb_list[-1][1] - print(f'Error getting board info: {e}, line {line_number}') + print(f'Error getting board data: {e}, line {line_number}') return {} print(f'Curation stats for items (and versions) published {f.get_cardlist_filter()['description']}') @@ -357,17 +419,56 @@ def run(args): + f'6M_med_items,6M_med_time ({args.units}),6M_hard_items,6M_hard_time ({args.units}),' + f'1Y_easy_items,1Y_easy_time ({args.units}),1Y_med_items,1Y_med_time ({args.units}),' + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_easy_items,2Y_easy_time ({args.units}),' - + f'2Y_med_items,2Y_med_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units})\n') + + f'2Y_med_items,2Y_med_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units}),' + + f'total_reviewer_1_items,total_reviewer_1_time ({args.units}),' + + f'total_reviewer_2_items,total_reviewer_2_time ({args.units}),' + + f'3M_reviewer_1_items,3M_reviewer_1_time ({args.units}),' + + f'3M_reviewer_2_items,3M_reviewer_2_time ({args.units}),' + + f'6M_reviewer_1_items,6M_reviewer_1_time ({args.units}),' + + f'6M_reviewer_2_items,6M_reviewer_2_time ({args.units}),' + + f'1Y_reviewer_1_items,1Y_reviewer_1_time ({args.units}),' + + f'1Y_reviewer_2_items,1Y_reviewer_2_time ({args.units}),' + + f'2Y_reviewer_1_items,2Y_reviewer_1_time ({args.units}),' + + f'2Y_reviewer_2_items,2Y_reviewer_2_time ({args.units}),' + + f'3M_easy_reviewer_1_items,3M_easy_reviewer_1_time ({args.units}),' + + f'3M_easy_reviewer_2_items,3M_easy_reviewer_2_time ({args.units}),' + + f'3M_med_reviewer_1_items,3M_med_reviewer_1_time ({args.units}),' + + f'3M_med_reviewer_2_items,3M_med_reviewer_2_time ({args.units}),' + + f'3M_hard_reviewer_1_items,3M_hard_reviewer_1_time ({args.units}),' + + f'3M_hard_reviewer_2_items,3M_hard_reviewer_2_time ({args.units}),' + + f'6M_easy_reviewer_1_items,6M_easy_reviewer_1_time ({args.units}),' + + f'6M_easy_reviewer_2_items,6M_easy_reviewer_2_time ({args.units}),' + + f'6M_med_reviewer_1_items,6M_med_reviewer_1_time ({args.units}),' + + f'6M_med_reviewer_2_items,6M_med_reviewer_2_time ({args.units}),' + + f'6M_hard_reviewer_1_items,6M_hard_reviewer_1_time ({args.units}),' + + f'6M_hard_reviewer_2_items,6M_hard_reviewer_2_time ({args.units}),' + + f'1Y_easy_reviewer_1_items,1Y_easy_reviewer_1_time ({args.units}),' + + f'1Y_easy_reviewer_2_items,1Y_easy_reviewer_2_time ({args.units}),' + + f'1Y_med_reviewer_1_items,1Y_med_reviewer_1_time ({args.units}),' + + f'1Y_med_reviewer_2_items,1Y_med_reviewer_2_time ({args.units}),' + + f'1Y_hard_reviewer_1_items,1Y_hard_reviewer_1_time ({args.units}),' + + f'1Y_hard_reviewer_2_items,1Y_hard_reviewer_2_time ({args.units}),' + + f'2Y_easy_reviewer_1_items,2Y_easy_reviewer_1_time ({args.units}),' + + f'2Y_easy_reviewer_2_items,2Y_easy_reviewer_2_time ({args.units}),' + + f'2Y_med_reviewer_1_items,2Y_med_reviewer_1_time ({args.units}),' + + f'2Y_med_reviewer_2_items,2Y_med_reviewer_2_time ({args.units}),' + + f'2Y_hard_reviewer_1_items,2Y_hard_reviewer_1_time ({args.units}),' + + f'2Y_hard_reviewer_2_items,2Y_hard_reviewer_2_time ({args.units}),' + + '\n') else: - print(f'username\ttotal_items\ttotal_time ({args.units})\tlast 3m items\tlast 3m time ({args.units})') + print( + f'username\ttot_itms\ttot_tm({args.units})\tlast 3m itms\tlast 3m tm({args.units})' + + f' rvwr1_itms rvwr1_tm({args.units})' + + f' rvwr2_itms rvwr2_tm({args.units})') total_time = 0 total_items = 0 for username, curator in curators.items(): if outfile: - s = ('{0},{1},{2},' # username, total items, total time + s = ( + '{0},{1},{2},' # username, total items, total time + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + '{9},{10},{11},{12},' # 3M, 6M items & time + '{13},{14},{15},{16},' # 1Y, 2Y items & time @@ -376,11 +477,30 @@ def run(args): + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time - + '{37},{38},{39},{40}') # 2Y_med, 2Y_hard items & time + + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time + + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time + + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time + + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time + + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time + + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time + + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time + + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time + + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time + + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time + + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time + + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time + + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time + + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time + + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time + + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time + + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time + + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time + ) else: s = ('{0} \t' + '{1} \t\t {2} \t\t' - + '{9} \t\t {10} \t') + + '{9} \t\t {10} \t\t' + + '{41}\t {42}\t\t{43}\t{44}') s = s.format( username, # 0 @@ -403,9 +523,43 @@ def run(args): curator['1Y_hard_items'], f.format_duration(str(curator['1Y_hard_time'])+'s', args.units), # 33,34 curator['2Y_easy_items'], f.format_duration(str(curator['2Y_easy_time'])+'s', args.units), # 35,36 curator['2Y_med_items'], f.format_duration(str(curator['2Y_med_time'])+'s', args.units), # 37,38 - curator['2Y_hard_items'], f.format_duration(str(curator['2Y_hard_time'])+'s', args.units)) # 39,40 - - + curator['2Y_hard_items'], f.format_duration(str(curator['2Y_hard_time'])+'s', args.units), # 39,40 + curator['total_reviewer1_items'], f.format_duration(str(curator['total_reviewer1_time'])+'s', args.units), # 41,42 + curator['total_reviewer2_items'], f.format_duration(str(curator['total_reviewer2_time'])+'s', args.units), # 43,44 + curator['3M_reviewer1_items'], f.format_duration(str(curator['3M_reviewer1_time'])+'s', args.units), # 45,46 + curator['3M_reviewer2_items'], f.format_duration(str(curator['3M_reviewer2_time'])+'s', args.units), # 47,48 + curator['6M_reviewer1_items'], f.format_duration(str(curator['6M_reviewer1_time'])+'s', args.units), # 49,50 + curator['6M_reviewer2_items'], f.format_duration(str(curator['6M_reviewer2_time'])+'s', args.units), # 51,52 + curator['1Y_reviewer1_items'], f.format_duration(str(curator['1Y_reviewer1_time'])+'s', args.units), # 53,54 + curator['1Y_reviewer2_items'], f.format_duration(str(curator['1Y_reviewer2_time'])+'s', args.units), # 55,56 + curator['2Y_reviewer1_items'], f.format_duration(str(curator['2Y_reviewer1_time'])+'s', args.units), # 57,58 + curator['2Y_reviewer2_items'], f.format_duration(str(curator['2Y_reviewer2_time'])+'s', args.units), # 59,60 + curator['3M_easy_reviewer1_items'], f.format_duration(str(curator['3M_easy_reviewer1_time'])+'s', args.units), # 61,62 + curator['3M_easy_reviewer2_items'], f.format_duration(str(curator['3M_easy_reviewer2_time'])+'s', args.units), # 63,64 + curator['3M_med_reviewer1_items'], f.format_duration(str(curator['3M_med_reviewer1_time'])+'s', args.units), # 65,66 + curator['3M_med_reviewer2_items'], f.format_duration(str(curator['3M_med_reviewer2_time'])+'s', args.units), # 67,68 + curator['3M_hard_reviewer1_items'], f.format_duration(str(curator['3M_hard_reviewer1_time'])+'s', args.units), # 69,70 + curator['3M_hard_reviewer2_items'], f.format_duration(str(curator['3M_hard_reviewer2_time'])+'s', args.units), # 71,72 + curator['6M_easy_reviewer1_items'], f.format_duration(str(curator['6M_easy_reviewer1_time'])+'s', args.units), # 73,74 + curator['6M_easy_reviewer2_items'], f.format_duration(str(curator['6M_easy_reviewer2_time'])+'s', args.units), # 75,76 + curator['6M_med_reviewer1_items'], f.format_duration(str(curator['6M_med_reviewer1_time'])+'s', args.units), # 77,78 + curator['6M_med_reviewer2_items'], f.format_duration(str(curator['6M_med_reviewer2_time'])+'s', args.units), # 79,80 + curator['6M_hard_reviewer1_items'], f.format_duration(str(curator['6M_hard_reviewer1_time'])+'s', args.units), # 81,82 + curator['6M_hard_reviewer2_items'], f.format_duration(str(curator['6M_hard_reviewer2_time'])+'s', args.units), # 83,84 + curator['1Y_easy_reviewer1_items'], f.format_duration(str(curator['1Y_easy_reviewer1_time'])+'s', args.units), # 85,86 + curator['1Y_easy_reviewer2_items'], f.format_duration(str(curator['1Y_easy_reviewer2_time'])+'s', args.units), # 87,88 + curator['1Y_med_reviewer1_items'], f.format_duration(str(curator['1Y_med_reviewer1_time'])+'s', args.units), # 89,90 + curator['1Y_med_reviewer2_items'], f.format_duration(str(curator['1Y_med_reviewer2_time'])+'s', args.units), # 91,92 + curator['1Y_hard_reviewer1_items'], f.format_duration(str(curator['1Y_hard_reviewer1_time'])+'s', args.units), # 93,94 + curator['1Y_hard_reviewer2_items'], f.format_duration(str(curator['1Y_hard_reviewer2_time'])+'s', args.units), # 95,96 + curator['2Y_easy_reviewer1_items'], f.format_duration(str(curator['2Y_easy_reviewer1_time'])+'s', args.units), # 97,98 + curator['2Y_easy_reviewer2_items'], f.format_duration(str(curator['2Y_easy_reviewer2_time'])+'s', args.units), # 99,100 + curator['2Y_med_reviewer1_items'], f.format_duration(str(curator['2Y_med_reviewer1_time'])+'s', args.units), # 101,102 + curator['2Y_med_reviewer2_items'], f.format_duration(str(curator['2Y_med_reviewer2_time'])+'s', args.units), # 103,104 + curator['2Y_hard_reviewer1_items'], f.format_duration(str(curator['2Y_hard_reviewer1_time'])+'s', args.units), # 105,106 + curator['2Y_hard_reviewer2_items'], f.format_duration(str(curator['2Y_hard_reviewer2_time'])+'s', args.units) # 107,108 + ) + total_time += curator['total_time'] total_items += curator['total_items'] diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py index ab9609a..839eea6 100644 --- a/packages/trello_reports/run/main.py +++ b/packages/trello_reports/run/main.py @@ -74,7 +74,6 @@ def run(args): environ['API_TRELLO_KEY'] = secrets.api_trello_key environ['API_TRELLO_TOKEN'] = secrets.api_trello_token environ['TRELLO_BOARD_ID'] = secrets.trello_board_id - environ['TRELLO_PUBLISHEDLIST_ID'] = secrets.trello_publishedlist_id environ['GSHEETS_DASHBOARD_POST_URL'] = secrets.gsheets_dashboard_post_url environ['GSHEETS_DASHBOARD_KEY'] = secrets.gsheets_dashboard_key environ['TOKEN'] = secrets.do_token From 16485fc67d82500f540052c3e737f626d28933eb Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 10:43:14 -0400 Subject: [PATCH 12/20] Add check for number of cards returned from Trello The API returns a max of 1000 cards which requires pagination which is not implemented yet --- packages/trello_reports/run/curator_report.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index 2e1762f..fb6df1b 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -17,7 +17,7 @@ sys.path.insert(0, 'lib/') import functions as f -debug = 0 # 0 = off, 1 = print card name + output cards to cards.csv, 2 = 1 + print more card info, 3 = 2 + card filter results +debug = 0 # 0 = off, 1 = print card name + output cards to cards.csv, 2 = 1 + print more card info, 3 = 2 + card filter results tc = None currdate = datetime.now(tzlocal()) @@ -33,11 +33,14 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): curators = [] - if debug > 0: print(card.name) + if debug > 0: + print(card.name) for field in card.custom_fields: if 'reviewer_' in field.name: - if debug > 1: print(f' {field.name}: {field.value}') + if debug > 1: + print(f' {field.name}: {field.value}') + curators.append(field.value) if type(existing_curators) is dict and field.value not in existing_curators: @@ -45,11 +48,11 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): # The value of the reviewer_x custom field in trello must be a trello username existing_curators[field.value] = { 'id': '', 'total_items': 0, 'total_time': 0, - 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0,'hard_items': 0, 'hard_time': 0, + 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0, 'hard_items': 0, 'hard_time': 0, '3M_items': 0, '3M_time': 0, '6M_items': 0, '6M_time': 0, '1Y_items': 0, '1Y_time': 0, - '2Y_items' :0, '2Y_time': 0, + '2Y_items': 0, '2Y_time': 0, '3M_easy_items': 0, '3M_easy_time': 0, '3M_med_items': 0, '3M_med_time': 0, '3M_hard_items': 0, '3M_hard_time': 0, @@ -118,7 +121,7 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): return curators -def get_card_time(card, userid: str=None): +def get_card_time(card, userid: str = None): """ Returns the total time in seconds from the Activity powerup for this card. If userid is given, limits the time to only the given userid @@ -137,7 +140,7 @@ def get_card_time(card, userid: str=None): return total -def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: str, card, cardfilter: dict[str, str]=None): +def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: str, card, cardfilter: dict[str, str] = None): """ For the given curators dictionary as generated by get_card_curators @@ -149,7 +152,8 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: existing_curators is a dict set up by get_card_curators """ - if debug > 2: print(f'cardfilter: {cardfilter}') + if debug > 2: + print(f'cardfilter: {cardfilter}') # Turn the custom fields into a dict for ease of use customfields = {} @@ -207,7 +211,7 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: curator[curators_field_prefix + '_items'] += 1 # Now, add to the specific reviewer 1 or 2 counts for the given curator - # and record whether that curator was reviewer 1 or 2. + # and record whether that curator was reviewer 1 or 2. if 'reviewer_1' in fieldname and username in fieldvalue: curator[curators_field_prefix + '_reviewer1_items'] += 1 reviewer1_id = curator['id'] @@ -338,6 +342,9 @@ def run(args): board = tc.get_board(environ['TRELLO_BOARD_ID']) cards = board.get_cards(filters=f.get_cardlist_filter()['query']) + if len(cards) >= 1000: + raise Exception("Trello API returned 1000 cards. Pagination may be needed but is not yet implemented") + # Preprocess board cards to filter out the ones without published_date set. # Do it this way instead of by returning only cards from the Published list so that # we grab cards in other lists that may have been published already but have been moved to another list. From 38f38b94fe08dcc5df40b0b2788a5ea45e9bbac5 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 14:16:41 -0400 Subject: [PATCH 13/20] Implement sending `curators` report to gsheet Increase python version to 3.11 (highest available on DO without needing to install custom). Changes were needed to support it (mainly f-strings which were using 3.12 features) --- gsheet_webapp/Code.gs | 13 +- gsheet_webapp/README.md | 1 + gsheet_webapp/Tests.gs | 145 +++++++++++++++++- lib/functions.py | 5 +- packages/trello_reports/run/curator_report.py | 120 ++++++++------- packages/trello_reports/run/main.py | 9 +- project.yml | 4 +- 7 files changed, 226 insertions(+), 71 deletions(-) diff --git a/gsheet_webapp/Code.gs b/gsheet_webapp/Code.gs index fae4ba5..bf3231c 100644 --- a/gsheet_webapp/Code.gs +++ b/gsheet_webapp/Code.gs @@ -33,8 +33,9 @@ function test(){ curl -L "script url" -H "Content-Type: application/json" --data '{"action":"insertupdate"}' */ - e=testData_insertExists_users(); + //e=testData_insertExists_users(); //e=testData_insertExists_items(); + e=testData_insertExists_curators(); console.log(doPost(e).getContent()); } @@ -123,14 +124,18 @@ function insert_or_update(data, sheet_name, headRow = 1) { if (headers[i] == "Timestamp"){ // special case if you include a 'Timestamp' column row.push(new Date()); } else { // else use header name to get data - row.push(item[headers[i]]); + row.push(item[headers[i].trim()]); } } rows.push(row); } + // clear sheet before adding values + if(sheet.getLastRow()>1){ + sheet.deleteRows(2, sheet.getLastRow()-1); + } + // more efficient to set values as [][] array than individually - sheet.deleteRows(2, sheet.getLastRow()-1); sheet.getRange(sheet.getLastRow()+1, 1, rows.length, rows[0].length).setValues(rows); SpreadsheetApp.flush(); @@ -153,4 +158,4 @@ function insert_or_update(data, sheet_name, headRow = 1) { function setup() { var doc = SpreadsheetApp.getActiveSpreadsheet(); SCRIPT_PROP.setProperty("spreadsheet_id", doc.getId()); -} \ No newline at end of file +} diff --git a/gsheet_webapp/README.md b/gsheet_webapp/README.md index 8195022..2d096b6 100644 --- a/gsheet_webapp/README.md +++ b/gsheet_webapp/README.md @@ -9,6 +9,7 @@ This Sheet accepts data from the report generator via a POST and stores it for u 1. `Readme` (optional) 1. `items`. Create columns exactly matching the name and order of the keys in the `articles` dictionary from [`items_report.py`](../packages/redata_reports/run/items_report.py) 1. `users`. Create columns exactly matching the name and order of the keys in the `account_info` dictionary from [`userquota_report.py`](../packages/redata_reports/run/userquota_report.py) + 1. `curators`. Create columns exactly matching the name and order of the `curators` dictionary from [`curators_report.py`](../packages/trello_reports/run/curators_report.py) ## WebApp diff --git a/gsheet_webapp/Tests.gs b/gsheet_webapp/Tests.gs index b513d88..c758f13 100644 --- a/gsheet_webapp/Tests.gs +++ b/gsheet_webapp/Tests.gs @@ -55,7 +55,7 @@ function testData_insertExists_items(){ } ) } - }; + } } function testData_insertExists_users(){ @@ -93,5 +93,146 @@ function testData_insertExists_users(){ } ) } - }; + } } + +function testData_insertExists_curators(){ + return { + "postData": { + "contents": JSON.stringify( + { + "action":"insertupdate", + "accesskey": PropertiesService.getScriptProperties().getProperty("accesskey"), + "sheet": "curators", + "data": + [ + { + "username": "hadepoju", + "id": "65a860d9a082cedfebebe8de", + "total_items": 3, + "total_time": 14707.716, + "easy_items": 1, + "easy_time": 938.044, + "med_items": 1, + "med_time": 1939.21, + "hard_items": 1, + "hard_time": 11830.462, + "3M_items": 3, + "3M_time": 14707.716, + "6M_items": 3, + "6M_time": 14707.716, + "1Y_items": 3, + "1Y_time": 14707.716, + "2Y_items": 3, + "2Y_time": 14707.716, + "3M_easy_items": 1, + "3M_easy_time": 938.044, + "3M_med_items": 1, + "3M_med_time": 1939.21, + "3M_hard_items": 1, + "3M_hard_time": 11830.462, + "6M_easy_items": 1, + "6M_easy_time": 938.044, + "6M_med_items": 1, + "6M_med_time": 1939.21, + "6M_hard_items": 1, + "6M_hard_time": 11830.462, + "1Y_easy_items": 1, + "1Y_easy_time": 938.044, + "1Y_med_items": 1, + "1Y_med_time": 1939.21, + "1Y_hard_items": 1, + "1Y_hard_time": 11830.462, + "2Y_easy_items": 1, + "2Y_easy_time": 938.044, + "2Y_med_items": 1, + "2Y_med_time": 1939.21, + "2Y_hard_items": 1, + "2Y_hard_time": 11830.462, + "total_reviewer1_items": 1, + "total_reviewer1_time": 11830.462, + "total_reviewer2_items": 2, + "total_reviewer2_time": 2877.254, + "easy_reviewer1_items": 0, + "easy_reviewer1_time": 0, + "easy_reviewer2_items": 1, + "easy_reviewer2_time": 938.044, + "med_reviewer1_items": 0, + "med_reviewer1_time": 0, + "med_reviewer2_items": 1, + "med_reviewer2_time": 1939.21, + "hard_reviewer1_items": 1, + "hard_reviewer1_time": 11830.462, + "hard_reviewer2_items": 0, + "hard_reviewer2_time": 0, + "3M_reviewer1_items": 1, + "3M_reviewer1_time": 11830.462, + "3M_reviewer2_items": 2, + "3M_reviewer2_time": 2877.254, + "6M_reviewer1_items": 1, + "6M_reviewer1_time": 11830.462, + "6M_reviewer2_items": 2, + "6M_reviewer2_time": 2877.254, + "1Y_reviewer1_items": 1, + "1Y_reviewer1_time": 11830.462, + "1Y_reviewer2_items": 2, + "1Y_reviewer2_time": 2877.254, + "2Y_reviewer1_items": 1, + "2Y_reviewer1_time": 11830.462, + "2Y_reviewer2_items": 2, + "2Y_reviewer2_time": 2877.254, + "3M_easy_reviewer1_items": 0, + "3M_easy_reviewer1_time": 0, + "3M_easy_reviewer2_items": 1, + "3M_easy_reviewer2_time": 938.044, + "6M_easy_reviewer1_items": 0, + "6M_easy_reviewer1_time": 0, + "6M_easy_reviewer2_items": 1, + "6M_easy_reviewer2_time": 938.044, + "1Y_easy_reviewer1_items": 0, + "1Y_easy_reviewer1_time": 0, + "1Y_easy_reviewer2_items": 1, + "1Y_easy_reviewer2_time": 938.044, + "2Y_easy_reviewer1_items": 0, + "2Y_easy_reviewer1_time": 0, + "2Y_easy_reviewer2_items": 1, + "2Y_easy_reviewer2_time": 938.044, + "3M_med_reviewer1_items": 0, + "3M_med_reviewer1_time": 0, + "3M_med_reviewer2_items": 1, + "3M_med_reviewer2_time": 1939.21, + "6M_med_reviewer1_items": 0, + "6M_med_reviewer1_time": 0, + "6M_med_reviewer2_items": 1, + "6M_med_reviewer2_time": 1939.21, + "1Y_med_reviewer1_items": 0, + "1Y_med_reviewer1_time": 0, + "1Y_med_reviewer2_items": 1, + "1Y_med_reviewer2_time": 1939.21, + "2Y_med_reviewer1_items": 0, + "2Y_med_reviewer1_time": 0, + "2Y_med_reviewer2_items": 1, + "2Y_med_reviewer2_time": 1939.21, + "3M_hard_reviewer1_items": 1, + "3M_hard_reviewer1_time": 11830.462, + "3M_hard_reviewer2_items": 0, + "3M_hard_reviewer2_time": 0, + "6M_hard_reviewer1_items": 1, + "6M_hard_reviewer1_time": 11830.462, + "6M_hard_reviewer2_items": 0, + "6M_hard_reviewer2_time": 0, + "1Y_hard_reviewer1_items": 1, + "1Y_hard_reviewer1_time": 11830.462, + "1Y_hard_reviewer2_items": 0, + "1Y_hard_reviewer2_time": 0, + "2Y_hard_reviewer1_items": 1, + "2Y_hard_reviewer1_time": 11830.462, + "2Y_hard_reviewer2_items": 0, + "2Y_hard_reviewer2_time": 0 + } + ] + } + ) + } + } +} \ No newline at end of file diff --git a/lib/functions.py b/lib/functions.py index c18e6a6..df16c5a 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -41,7 +41,10 @@ def get_cardlist_filter(): def sync_to_dashboard(data, report): """ - Sends the given data to the Google Sheet given by report via the sheet's POST request url + Sends the given data to the Google Sheet given by `report` via the sheet's POST request url. + + data: a list of dictionaries corresponding to 1 row of data. + report: the name of the sheet to put the data. """ print(f'Sending data for report "{report}" to dashboard') diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index fb6df1b..1156980 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -35,7 +35,7 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): if debug > 0: print(card.name) - + for field in card.custom_fields: if 'reviewer_' in field.name: if debug > 1: @@ -47,7 +47,7 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): # Add a new curator to the list if they're not already there # The value of the reviewer_x custom field in trello must be a trello username existing_curators[field.value] = { - 'id': '', 'total_items': 0, 'total_time': 0, + 'username': field.value, 'id': '', 'total_items': 0, 'total_time': 0, 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0, 'hard_items': 0, 'hard_time': 0, '3M_items': 0, '3M_time': 0, '6M_items': 0, '6M_time': 0, @@ -104,7 +104,8 @@ def get_card_curators(card, existing_curators: dict[str, any] = None): '1Y_hard_reviewer1_items': 0, '1Y_hard_reviewer1_time': 0, '1Y_hard_reviewer2_items': 0, '1Y_hard_reviewer2_time': 0, '2Y_hard_reviewer1_items': 0, '2Y_hard_reviewer1_time': 0, - '2Y_hard_reviewer2_items': 0, '2Y_hard_reviewer2_time': 0 + '2Y_hard_reviewer2_items': 0, '2Y_hard_reviewer2_time': 0, + 'report_date': f.get_report_date().strftime('%Y-%m-%d %H:%M:%S') } for member in tc.get_board(card.board_id).all_members(): @@ -175,7 +176,7 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: if debug > 2: print(f' {name} filter: result={filter_result[name]} | ' - + f'requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else ''}') + + f'requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else None }') case 'published_date': pubdate = '' cutoffdate = '' @@ -187,7 +188,7 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: if debug > 2: print(f' {name} filter: result={filter_result[name]} | ' - + f'card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') + + f'card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') case 'reviewer_1' | 'reviewer_2': if name in customfields.keys(): filter_result[name] = True @@ -351,9 +352,11 @@ def run(args): publishedcards = [] fl = None - if debug > 0: fl = open('cards.csv', 'w', encoding="utf-8") + if debug > 0: + fl = open('cards.csv', 'w', encoding="utf-8") - if fl: fl.write( + if fl: + fl.write( 'name,difficulty,total_time_mins,article_version,article_revision,' + 'reviewer_1,reviewer_2,submission_date,forms_sent_date,reviewed_date,published_date,keep?\n') for card in cards: @@ -366,16 +369,18 @@ def run(args): if keep: publishedcards.append(card) - if fl: fl.write( - f'"{card.name.replace('"','""')}",' - + f'{customfields.get('difficulty', '""')},' - + f'{customfields.get('total_time_mins', '""')},' - + f'{customfields.get('article_version', '""')},{customfields.get('article_revision', '""')},' - + f'{customfields.get('reviewer_1', '""')},{customfields.get('reviewer_2', '""')},' - + f'{customfields.get('submission_date', '""')},{customfields.get('forms_sent_date', '""')},' - + f'{customfields.get('reviewed_date', '""')},{customfields.get('published_date', '""')},' - + f'{keep}\n') - if fl: fl.close() + if fl: + fl.write( + '"' + card.name.replace('"', '""') + '",' + + customfields.get('difficulty', '""') + ',' + + str(customfields.get('total_time_mins', '""')) + ',' + + str(customfields.get('article_version', '""')) + ',' + str(customfields.get('article_revision', '""')) + ',' + + str(customfields.get('reviewer_1', '""')) + ',' + str(customfields.get('reviewer_2', '""')) + ',' + + str(customfields.get('submission_date', '""')) + ',' + str(customfields.get('forms_sent_date', '""')) + ',' + + str(customfields.get('reviewed_date', '""')) + ',' + str(customfields.get('published_date', '""')) + ',' + + f'{keep}\n') + if fl: + fl.close() for card in publishedcards: # Preprocess the card to get the reviewers @@ -403,14 +408,13 @@ def run(args): curator_2y_hard_items_time(curators, card) print() - except Exception as e: tb_list = traceback.extract_tb(sys.exc_info()[2]) line_number = tb_list[-1][1] print(f'Error getting board data: {e}, line {line_number}') return {} - print(f'Curation stats for items (and versions) published {f.get_cardlist_filter()['description']}') + print(f'Curation stats for items (and versions) published {f.get_cardlist_filter()["description"]}') print(f'Processed {len(publishedcards)} cards with published_date set, out of {len(cards)} fetched') outfile = None @@ -427,43 +431,43 @@ def run(args): + f'1Y_easy_items,1Y_easy_time ({args.units}),1Y_med_items,1Y_med_time ({args.units}),' + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_easy_items,2Y_easy_time ({args.units}),' + f'2Y_med_items,2Y_med_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units}),' - + f'total_reviewer_1_items,total_reviewer_1_time ({args.units}),' - + f'total_reviewer_2_items,total_reviewer_2_time ({args.units}),' - + f'3M_reviewer_1_items,3M_reviewer_1_time ({args.units}),' - + f'3M_reviewer_2_items,3M_reviewer_2_time ({args.units}),' - + f'6M_reviewer_1_items,6M_reviewer_1_time ({args.units}),' - + f'6M_reviewer_2_items,6M_reviewer_2_time ({args.units}),' - + f'1Y_reviewer_1_items,1Y_reviewer_1_time ({args.units}),' - + f'1Y_reviewer_2_items,1Y_reviewer_2_time ({args.units}),' - + f'2Y_reviewer_1_items,2Y_reviewer_1_time ({args.units}),' - + f'2Y_reviewer_2_items,2Y_reviewer_2_time ({args.units}),' - + f'3M_easy_reviewer_1_items,3M_easy_reviewer_1_time ({args.units}),' - + f'3M_easy_reviewer_2_items,3M_easy_reviewer_2_time ({args.units}),' - + f'3M_med_reviewer_1_items,3M_med_reviewer_1_time ({args.units}),' - + f'3M_med_reviewer_2_items,3M_med_reviewer_2_time ({args.units}),' - + f'3M_hard_reviewer_1_items,3M_hard_reviewer_1_time ({args.units}),' - + f'3M_hard_reviewer_2_items,3M_hard_reviewer_2_time ({args.units}),' - + f'6M_easy_reviewer_1_items,6M_easy_reviewer_1_time ({args.units}),' - + f'6M_easy_reviewer_2_items,6M_easy_reviewer_2_time ({args.units}),' - + f'6M_med_reviewer_1_items,6M_med_reviewer_1_time ({args.units}),' - + f'6M_med_reviewer_2_items,6M_med_reviewer_2_time ({args.units}),' - + f'6M_hard_reviewer_1_items,6M_hard_reviewer_1_time ({args.units}),' - + f'6M_hard_reviewer_2_items,6M_hard_reviewer_2_time ({args.units}),' - + f'1Y_easy_reviewer_1_items,1Y_easy_reviewer_1_time ({args.units}),' - + f'1Y_easy_reviewer_2_items,1Y_easy_reviewer_2_time ({args.units}),' - + f'1Y_med_reviewer_1_items,1Y_med_reviewer_1_time ({args.units}),' - + f'1Y_med_reviewer_2_items,1Y_med_reviewer_2_time ({args.units}),' - + f'1Y_hard_reviewer_1_items,1Y_hard_reviewer_1_time ({args.units}),' - + f'1Y_hard_reviewer_2_items,1Y_hard_reviewer_2_time ({args.units}),' - + f'2Y_easy_reviewer_1_items,2Y_easy_reviewer_1_time ({args.units}),' - + f'2Y_easy_reviewer_2_items,2Y_easy_reviewer_2_time ({args.units}),' - + f'2Y_med_reviewer_1_items,2Y_med_reviewer_1_time ({args.units}),' - + f'2Y_med_reviewer_2_items,2Y_med_reviewer_2_time ({args.units}),' - + f'2Y_hard_reviewer_1_items,2Y_hard_reviewer_1_time ({args.units}),' - + f'2Y_hard_reviewer_2_items,2Y_hard_reviewer_2_time ({args.units}),' + + f'total_reviewer1_items,total_reviewer1_time ({args.units}),' + + f'total_reviewer2_items,total_reviewer2_time ({args.units}),' + + f'3M_reviewer1_items,3M_reviewer1_time ({args.units}),' + + f'3M_reviewer2_items,3M_reviewer2_time ({args.units}),' + + f'6M_reviewer1_items,6M_reviewer1_time ({args.units}),' + + f'6M_reviewer2_items,6M_reviewer2_time ({args.units}),' + + f'1Y_reviewer1_items,1Y_reviewer1_time ({args.units}),' + + f'1Y_reviewer2_items,1Y_reviewer2_time ({args.units}),' + + f'2Y_reviewer1_items,2Y_reviewer1_time ({args.units}),' + + f'2Y_reviewer2_items,2Y_reviewer2_time ({args.units}),' + + f'3M_easy_reviewer1_items,3M_easy_reviewer1_time ({args.units}),' + + f'3M_easy_reviewer2_items,3M_easy_reviewer2_time ({args.units}),' + + f'3M_med_reviewer1_items,3M_med_reviewer1_time ({args.units}),' + + f'3M_med_reviewer2_items,3M_med_reviewer2_time ({args.units}),' + + f'3M_hard_reviewer1_items,3M_hard_reviewer1_time ({args.units}),' + + f'3M_hard_reviewer2_items,3M_hard_reviewer2_time ({args.units}),' + + f'6M_easy_reviewer1_items,6M_easy_reviewer1_time ({args.units}),' + + f'6M_easy_reviewer2_items,6M_easy_reviewer2_time ({args.units}),' + + f'6M_med_reviewer1_items,6M_med_reviewer1_time ({args.units}),' + + f'6M_med_reviewer2_items,6M_med_reviewer2_time ({args.units}),' + + f'6M_hard_reviewer1_items,6M_hard_reviewer1_time ({args.units}),' + + f'6M_hard_reviewer2_items,6M_hard_reviewer2_time ({args.units}),' + + f'1Y_easy_reviewer1_items,1Y_easy_reviewer1_time ({args.units}),' + + f'1Y_easy_reviewer2_items,1Y_easy_reviewer2_time ({args.units}),' + + f'1Y_med_reviewer1_items,1Y_med_reviewer1_time ({args.units}),' + + f'1Y_med_reviewer2_items,1Y_med_reviewer2_time ({args.units}),' + + f'1Y_hard_reviewer1_items,1Y_hard_reviewer1_time ({args.units}),' + + f'1Y_hard_reviewer2_items,1Y_hard_reviewer2_time ({args.units}),' + + f'2Y_easy_reviewer1_items,2Y_easy_reviewer1_time ({args.units}),' + + f'2Y_easy_reviewer2_items,2Y_easy_reviewer2_time ({args.units}),' + + f'2Y_med_reviewer1_items,2Y_med_reviewer1_time ({args.units}),' + + f'2Y_med_reviewer2_items,2Y_med_reviewer2_time ({args.units}),' + + f'2Y_hard_reviewer1_items,2Y_hard_reviewer1_time ({args.units}),' + + f'2Y_hard_reviewer2_items,2Y_hard_reviewer2_time ({args.units}),' + '\n') - else: + elif not args.sync_to_dashboard: print( f'username\ttot_itms\ttot_tm({args.units})\tlast 3m itms\tlast 3m tm({args.units})' + f' rvwr1_itms rvwr1_tm({args.units})' @@ -475,7 +479,7 @@ def run(args): for username, curator in curators.items(): if outfile: s = ( - '{0},{1},{2},' # username, total items, total time + '{0},{1},{2},' # username, total items, total time + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + '{9},{10},{11},{12},' # 3M, 6M items & time + '{13},{14},{15},{16},' # 1Y, 2Y items & time @@ -576,10 +580,10 @@ def run(args): print(s) print() - print(f'Total curation hours:\t{f.format_duration(str(total_time)+'s', 'H')}') - print(f'Avg hours per item:\t{f.format_duration(str(total_time/total_items)+'s', 'H')}') + print(f'Total curation hours:\t{f.format_duration(str(total_time) + "s", "H")}') + print(f'Avg hours per item:\t{f.format_duration(str(total_time/total_items) + "s", "H")}') if outfile: outfile.close() - return curators + return list(curators.values()) diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py index 839eea6..ab007f3 100644 --- a/packages/trello_reports/run/main.py +++ b/packages/trello_reports/run/main.py @@ -57,12 +57,13 @@ def run(args): if 'curators' in args.report: print('Running "curators" report') data = curator_report.run(args) - result = result + f'Running "curator" report completed. {len(data)} curators.' + result = result + f'Running "curators" report completed. {len(data)} curators found.' if args.sync_to_dashboard: - result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curator")}.' + result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curators")}.' if 'items' in args.report: - print('Running "items" report') - result = 'not implemented' + print('Running "cards" report') + if args.sync_to_dashboard: + result = result + '\nSyncing "cards" to dashboard completed. Result: Not implemented.' return result diff --git a/project.yml b/project.yml index 276e705..dd6124b 100644 --- a/project.yml +++ b/project.yml @@ -2,7 +2,7 @@ packages: - name: redata_reports functions: - name: run - runtime: 'python:default' + runtime: 'python:3.11' web: true limits: timeout: 420000 @@ -16,7 +16,7 @@ packages: - name: trello_reports functions: - name: run - runtime: 'python:default' + runtime: 'python:3.11' web: true limits: timeout: 420000 From 1817268a0e46be9c010b2816e83b0a1b96b66e2b Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 14:55:46 -0400 Subject: [PATCH 14/20] lint --- packages/trello_reports/run/curator_report.py | 172 +++++++++--------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index 1156980..055dbb8 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -176,7 +176,7 @@ def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: if debug > 2: print(f' {name} filter: result={filter_result[name]} | ' - + f'requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else None }') + + f'requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else None}') case 'published_date': pubdate = '' cutoffdate = '' @@ -479,96 +479,96 @@ def run(args): for username, curator in curators.items(): if outfile: s = ( - '{0},{1},{2},' # username, total items, total time - + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time - + '{9},{10},{11},{12},' # 3M, 6M items & time - + '{13},{14},{15},{16},' # 1Y, 2Y items & time - + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time - + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time - + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time - + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time - + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time - + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time - + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time - + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time - + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time - + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time - + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time - + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time - + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time - + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time - + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time - + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time - + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time - + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time - + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time - + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time - + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time - + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time - + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time + '{0},{1},{2},' # username, total items, total time + + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + + '{9},{10},{11},{12},' # 3M, 6M items & time + + '{13},{14},{15},{16},' # 1Y, 2Y items & time + + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time + + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time + + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time + + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time + + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time + + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time + + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time + + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time + + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time + + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time + + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time + + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time + + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time + + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time + + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time + + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time + + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time + + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time + + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time + + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time + + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time + + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time + + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time ) else: s = ('{0} \t' - + '{1} \t\t {2} \t\t' - + '{9} \t\t {10} \t\t' - + '{41}\t {42}\t\t{43}\t{44}') + + '{1} \t\t {2} \t\t' + + '{9} \t\t {10} \t\t' + + '{41}\t {42}\t\t{43}\t{44}') s = s.format( - username, # 0 - curator['total_items'], f.format_duration(str(curator['total_time'])+'s', args.units), # 1,2 - curator['easy_items'], f.format_duration(str(curator['easy_time'])+'s', args.units), # 3,4 - curator['med_items'], f.format_duration(str(curator['med_time'])+'s', args.units), # 5,6 - curator['hard_items'], f.format_duration(str(curator['hard_time'])+'s', args.units), # 7,8 - curator['3M_items'], f.format_duration(str(curator['3M_time'])+'s', args.units), # 9,10 - curator['6M_items'], f.format_duration(str(curator['6M_time'])+'s', args.units), # 11,12 - curator['1Y_items'], f.format_duration(str(curator['1Y_time'])+'s', args.units), # 13,14 - curator['2Y_items'], f.format_duration(str(curator['2Y_time'])+'s', args.units), # 15,16 - curator['3M_easy_items'], f.format_duration(str(curator['3M_easy_time'])+'s', args.units), # 17,18 - curator['3M_med_items'], f.format_duration(str(curator['3M_med_time'])+'s', args.units), # 19,20 - curator['3M_hard_items'], f.format_duration(str(curator['3M_hard_time'])+'s', args.units), # 21,22 - curator['6M_easy_items'], f.format_duration(str(curator['6M_easy_time'])+'s', args.units), # 23,24 - curator['6M_med_items'], f.format_duration(str(curator['6M_med_time'])+'s', args.units), # 25,26 - curator['6M_hard_items'], f.format_duration(str(curator['6M_hard_time'])+'s', args.units), # 27,28 - curator['1Y_easy_items'], f.format_duration(str(curator['1Y_easy_time'])+'s', args.units), # 29,30 - curator['1Y_med_items'], f.format_duration(str(curator['1Y_med_time'])+'s', args.units), # 31,32 - curator['1Y_hard_items'], f.format_duration(str(curator['1Y_hard_time'])+'s', args.units), # 33,34 - curator['2Y_easy_items'], f.format_duration(str(curator['2Y_easy_time'])+'s', args.units), # 35,36 - curator['2Y_med_items'], f.format_duration(str(curator['2Y_med_time'])+'s', args.units), # 37,38 - curator['2Y_hard_items'], f.format_duration(str(curator['2Y_hard_time'])+'s', args.units), # 39,40 - curator['total_reviewer1_items'], f.format_duration(str(curator['total_reviewer1_time'])+'s', args.units), # 41,42 - curator['total_reviewer2_items'], f.format_duration(str(curator['total_reviewer2_time'])+'s', args.units), # 43,44 - curator['3M_reviewer1_items'], f.format_duration(str(curator['3M_reviewer1_time'])+'s', args.units), # 45,46 - curator['3M_reviewer2_items'], f.format_duration(str(curator['3M_reviewer2_time'])+'s', args.units), # 47,48 - curator['6M_reviewer1_items'], f.format_duration(str(curator['6M_reviewer1_time'])+'s', args.units), # 49,50 - curator['6M_reviewer2_items'], f.format_duration(str(curator['6M_reviewer2_time'])+'s', args.units), # 51,52 - curator['1Y_reviewer1_items'], f.format_duration(str(curator['1Y_reviewer1_time'])+'s', args.units), # 53,54 - curator['1Y_reviewer2_items'], f.format_duration(str(curator['1Y_reviewer2_time'])+'s', args.units), # 55,56 - curator['2Y_reviewer1_items'], f.format_duration(str(curator['2Y_reviewer1_time'])+'s', args.units), # 57,58 - curator['2Y_reviewer2_items'], f.format_duration(str(curator['2Y_reviewer2_time'])+'s', args.units), # 59,60 - curator['3M_easy_reviewer1_items'], f.format_duration(str(curator['3M_easy_reviewer1_time'])+'s', args.units), # 61,62 - curator['3M_easy_reviewer2_items'], f.format_duration(str(curator['3M_easy_reviewer2_time'])+'s', args.units), # 63,64 - curator['3M_med_reviewer1_items'], f.format_duration(str(curator['3M_med_reviewer1_time'])+'s', args.units), # 65,66 - curator['3M_med_reviewer2_items'], f.format_duration(str(curator['3M_med_reviewer2_time'])+'s', args.units), # 67,68 - curator['3M_hard_reviewer1_items'], f.format_duration(str(curator['3M_hard_reviewer1_time'])+'s', args.units), # 69,70 - curator['3M_hard_reviewer2_items'], f.format_duration(str(curator['3M_hard_reviewer2_time'])+'s', args.units), # 71,72 - curator['6M_easy_reviewer1_items'], f.format_duration(str(curator['6M_easy_reviewer1_time'])+'s', args.units), # 73,74 - curator['6M_easy_reviewer2_items'], f.format_duration(str(curator['6M_easy_reviewer2_time'])+'s', args.units), # 75,76 - curator['6M_med_reviewer1_items'], f.format_duration(str(curator['6M_med_reviewer1_time'])+'s', args.units), # 77,78 - curator['6M_med_reviewer2_items'], f.format_duration(str(curator['6M_med_reviewer2_time'])+'s', args.units), # 79,80 - curator['6M_hard_reviewer1_items'], f.format_duration(str(curator['6M_hard_reviewer1_time'])+'s', args.units), # 81,82 - curator['6M_hard_reviewer2_items'], f.format_duration(str(curator['6M_hard_reviewer2_time'])+'s', args.units), # 83,84 - curator['1Y_easy_reviewer1_items'], f.format_duration(str(curator['1Y_easy_reviewer1_time'])+'s', args.units), # 85,86 - curator['1Y_easy_reviewer2_items'], f.format_duration(str(curator['1Y_easy_reviewer2_time'])+'s', args.units), # 87,88 - curator['1Y_med_reviewer1_items'], f.format_duration(str(curator['1Y_med_reviewer1_time'])+'s', args.units), # 89,90 - curator['1Y_med_reviewer2_items'], f.format_duration(str(curator['1Y_med_reviewer2_time'])+'s', args.units), # 91,92 - curator['1Y_hard_reviewer1_items'], f.format_duration(str(curator['1Y_hard_reviewer1_time'])+'s', args.units), # 93,94 - curator['1Y_hard_reviewer2_items'], f.format_duration(str(curator['1Y_hard_reviewer2_time'])+'s', args.units), # 95,96 - curator['2Y_easy_reviewer1_items'], f.format_duration(str(curator['2Y_easy_reviewer1_time'])+'s', args.units), # 97,98 - curator['2Y_easy_reviewer2_items'], f.format_duration(str(curator['2Y_easy_reviewer2_time'])+'s', args.units), # 99,100 - curator['2Y_med_reviewer1_items'], f.format_duration(str(curator['2Y_med_reviewer1_time'])+'s', args.units), # 101,102 - curator['2Y_med_reviewer2_items'], f.format_duration(str(curator['2Y_med_reviewer2_time'])+'s', args.units), # 103,104 - curator['2Y_hard_reviewer1_items'], f.format_duration(str(curator['2Y_hard_reviewer1_time'])+'s', args.units), # 105,106 - curator['2Y_hard_reviewer2_items'], f.format_duration(str(curator['2Y_hard_reviewer2_time'])+'s', args.units) # 107,108 + username, # 0 + curator['total_items'], f.format_duration(str(curator['total_time']) + 's', args.units), # 1,2 + curator['easy_items'], f.format_duration(str(curator['easy_time']) + 's', args.units), # 3,4 + curator['med_items'], f.format_duration(str(curator['med_time']) + 's', args.units), # 5,6 + curator['hard_items'], f.format_duration(str(curator['hard_time']) + 's', args.units), # 7,8 + curator['3M_items'], f.format_duration(str(curator['3M_time']) + 's', args.units), # 9,10 + curator['6M_items'], f.format_duration(str(curator['6M_time']) + 's', args.units), # 11,12 + curator['1Y_items'], f.format_duration(str(curator['1Y_time']) + 's', args.units), # 13,14 + curator['2Y_items'], f.format_duration(str(curator['2Y_time']) + 's', args.units), # 15,16 + curator['3M_easy_items'], f.format_duration(str(curator['3M_easy_time']) + 's', args.units), # 17,18 + curator['3M_med_items'], f.format_duration(str(curator['3M_med_time']) + 's', args.units), # 19,20 + curator['3M_hard_items'], f.format_duration(str(curator['3M_hard_time']) + 's', args.units), # 21,22 + curator['6M_easy_items'], f.format_duration(str(curator['6M_easy_time']) + 's', args.units), # 23,24 + curator['6M_med_items'], f.format_duration(str(curator['6M_med_time']) + 's', args.units), # 25,26 + curator['6M_hard_items'], f.format_duration(str(curator['6M_hard_time']) + 's', args.units), # 27,28 + curator['1Y_easy_items'], f.format_duration(str(curator['1Y_easy_time']) + 's', args.units), # 29,30 + curator['1Y_med_items'], f.format_duration(str(curator['1Y_med_time']) + 's', args.units), # 31,32 + curator['1Y_hard_items'], f.format_duration(str(curator['1Y_hard_time']) + 's', args.units), # 33,34 + curator['2Y_easy_items'], f.format_duration(str(curator['2Y_easy_time']) + 's', args.units), # 35,36 + curator['2Y_med_items'], f.format_duration(str(curator['2Y_med_time']) + 's', args.units), # 37,38 + curator['2Y_hard_items'], f.format_duration(str(curator['2Y_hard_time']) + 's', args.units), # 39,40 + curator['total_reviewer1_items'], f.format_duration(str(curator['total_reviewer1_time']) + 's', args.units), # 41,42 + curator['total_reviewer2_items'], f.format_duration(str(curator['total_reviewer2_time']) + 's', args.units), # 43,44 + curator['3M_reviewer1_items'], f.format_duration(str(curator['3M_reviewer1_time']) + 's', args.units), # 45,46 + curator['3M_reviewer2_items'], f.format_duration(str(curator['3M_reviewer2_time']) + 's', args.units), # 47,48 + curator['6M_reviewer1_items'], f.format_duration(str(curator['6M_reviewer1_time']) + 's', args.units), # 49,50 + curator['6M_reviewer2_items'], f.format_duration(str(curator['6M_reviewer2_time']) + 's', args.units), # 51,52 + curator['1Y_reviewer1_items'], f.format_duration(str(curator['1Y_reviewer1_time']) + 's', args.units), # 53,54 + curator['1Y_reviewer2_items'], f.format_duration(str(curator['1Y_reviewer2_time']) + 's', args.units), # 55,56 + curator['2Y_reviewer1_items'], f.format_duration(str(curator['2Y_reviewer1_time']) + 's', args.units), # 57,58 + curator['2Y_reviewer2_items'], f.format_duration(str(curator['2Y_reviewer2_time']) + 's', args.units), # 59,60 + curator['3M_easy_reviewer1_items'], f.format_duration(str(curator['3M_easy_reviewer1_time']) + 's', args.units), # 61,62 + curator['3M_easy_reviewer2_items'], f.format_duration(str(curator['3M_easy_reviewer2_time']) + 's', args.units), # 63,64 + curator['3M_med_reviewer1_items'], f.format_duration(str(curator['3M_med_reviewer1_time']) + 's', args.units), # 65,66 + curator['3M_med_reviewer2_items'], f.format_duration(str(curator['3M_med_reviewer2_time']) + 's', args.units), # 67,68 + curator['3M_hard_reviewer1_items'], f.format_duration(str(curator['3M_hard_reviewer1_time']) + 's', args.units), # 69,70 + curator['3M_hard_reviewer2_items'], f.format_duration(str(curator['3M_hard_reviewer2_time']) + 's', args.units), # 71,72 + curator['6M_easy_reviewer1_items'], f.format_duration(str(curator['6M_easy_reviewer1_time']) + 's', args.units), # 73,74 + curator['6M_easy_reviewer2_items'], f.format_duration(str(curator['6M_easy_reviewer2_time']) + 's', args.units), # 75,76 + curator['6M_med_reviewer1_items'], f.format_duration(str(curator['6M_med_reviewer1_time']) + 's', args.units), # 77,78 + curator['6M_med_reviewer2_items'], f.format_duration(str(curator['6M_med_reviewer2_time']) + 's', args.units), # 79,80 + curator['6M_hard_reviewer1_items'], f.format_duration(str(curator['6M_hard_reviewer1_time']) + 's', args.units), # 81,82 + curator['6M_hard_reviewer2_items'], f.format_duration(str(curator['6M_hard_reviewer2_time']) + 's', args.units), # 83,84 + curator['1Y_easy_reviewer1_items'], f.format_duration(str(curator['1Y_easy_reviewer1_time']) + 's', args.units), # 85,86 + curator['1Y_easy_reviewer2_items'], f.format_duration(str(curator['1Y_easy_reviewer2_time']) + 's', args.units), # 87,88 + curator['1Y_med_reviewer1_items'], f.format_duration(str(curator['1Y_med_reviewer1_time']) + 's', args.units), # 89,90 + curator['1Y_med_reviewer2_items'], f.format_duration(str(curator['1Y_med_reviewer2_time']) + 's', args.units), # 91,92 + curator['1Y_hard_reviewer1_items'], f.format_duration(str(curator['1Y_hard_reviewer1_time']) + 's', args.units), # 93,94 + curator['1Y_hard_reviewer2_items'], f.format_duration(str(curator['1Y_hard_reviewer2_time']) + 's', args.units), # 95,96 + curator['2Y_easy_reviewer1_items'], f.format_duration(str(curator['2Y_easy_reviewer1_time']) + 's', args.units), # 97,98 + curator['2Y_easy_reviewer2_items'], f.format_duration(str(curator['2Y_easy_reviewer2_time']) + 's', args.units), # 99,100 + curator['2Y_med_reviewer1_items'], f.format_duration(str(curator['2Y_med_reviewer1_time']) + 's', args.units), # 101,102 + curator['2Y_med_reviewer2_items'], f.format_duration(str(curator['2Y_med_reviewer2_time']) + 's', args.units), # 103,104 + curator['2Y_hard_reviewer1_items'], f.format_duration(str(curator['2Y_hard_reviewer1_time']) + 's', args.units), # 105,106 + curator['2Y_hard_reviewer2_items'], f.format_duration(str(curator['2Y_hard_reviewer2_time']) + 's', args.units) # 107,108 ) total_time += curator['total_time'] From a0f447d6b4f9df091f2cc81dd3dd6943af2d1b96 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 14:58:05 -0400 Subject: [PATCH 15/20] lint --- packages/trello_reports/run/curator_report.py | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index 055dbb8..1e79724 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -479,33 +479,33 @@ def run(args): for username, curator in curators.items(): if outfile: s = ( - '{0},{1},{2},' # username, total items, total time - + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time - + '{9},{10},{11},{12},' # 3M, 6M items & time - + '{13},{14},{15},{16},' # 1Y, 2Y items & time - + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time - + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time - + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time - + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time - + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time - + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time - + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time - + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time - + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time - + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time - + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time - + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time - + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time - + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time - + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time - + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time - + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time - + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time - + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time - + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time - + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time - + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time - + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time + '{0},{1},{2},' # username, total items, total time + + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + + '{9},{10},{11},{12},' # 3M, 6M items & time + + '{13},{14},{15},{16},' # 1Y, 2Y items & time + + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time + + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time + + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time + + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time + + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time + + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time + + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time + + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time + + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time + + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time + + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time + + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time + + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time + + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time + + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time + + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time + + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time + + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time + + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time + + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time + + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time + + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time + + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time ) else: s = ('{0} \t' @@ -581,7 +581,7 @@ def run(args): print() print(f'Total curation hours:\t{f.format_duration(str(total_time) + "s", "H")}') - print(f'Avg hours per item:\t{f.format_duration(str(total_time/total_items) + "s", "H")}') + print(f'Avg hours per item:\t{f.format_duration(str(total_time / total_items) + "s", "H")}') if outfile: outfile.close() From a1ba3e22982ac9211741468f2bcadf00ad42bba1 Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 15:00:03 -0400 Subject: [PATCH 16/20] lint --- packages/trello_reports/run/curator_report.py | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py index 1e79724..7b41d42 100644 --- a/packages/trello_reports/run/curator_report.py +++ b/packages/trello_reports/run/curator_report.py @@ -479,34 +479,34 @@ def run(args): for username, curator in curators.items(): if outfile: s = ( - '{0},{1},{2},' # username, total items, total time - + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time - + '{9},{10},{11},{12},' # 3M, 6M items & time - + '{13},{14},{15},{16},' # 1Y, 2Y items & time - + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time - + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time - + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time - + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time - + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time - + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time - + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time - + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time - + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time - + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time - + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time - + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time - + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time - + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time - + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time - + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time - + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time - + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time - + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time - + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time - + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time - + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time - + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time - ) + '{0},{1},{2},' # username, total items, total time + + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + + '{9},{10},{11},{12},' # 3M, 6M items & time + + '{13},{14},{15},{16},' # 1Y, 2Y items & time + + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time + + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time + + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time + + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time + + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time + + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time + + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time + + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time + + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time + + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time + + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time + + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time + + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time + + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time + + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time + + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time + + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time + + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time + + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time + + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time + + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time + + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time + + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time + ) else: s = ('{0} \t' + '{1} \t\t {2} \t\t' From 958c649ba8f820d5072c371a870d9c8e9f45eb7b Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Tue, 29 Jul 2025 15:09:21 -0400 Subject: [PATCH 17/20] Update README.md --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b7f798..c89cd66 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,12 @@ Show the command line options by running python ./packages/redata_reports/run/main.py -h ``` -Prior to generating any reports, configure API endpoints and keys. Copy [`./packages/redata_reports/run/secrets.example.py`](packages/redata_reports/run/secrets.example.py) to `./packages/redata_reports/run/secrets.py` and edit the fields with the appropriate values. See the comments in that file for specific instructions. To generate reports locally, only the Figshare API credentials are needed. +or +``` +python ./packages/trello_reports/run/main.py -h +``` + +Prior to generating any reports, configure API endpoints and keys. Copy [`./lib/secrets.example.py`](lib/secrets.example.py) to `./lib/secrets.py` and edit the fields with the appropriate values. See the comments in that file for specific instructions. To generate reports locally, Google Sheets and DigitalOcean credentials are not needed. Generate a report. E.g the users report. A CSV will be output in the current working directory in this example. ``` @@ -30,7 +35,7 @@ python ./packages/redata_reports/run/main.py -r users -o The `--sync-to-dashboard` option uploads data to the Google dashboard (data is stored in a Google Sheet and displayed in a Looker Studio dashboard). Setting this flag includes `-u B -r items -r users`. The destination sheet is configured in `secrets.py`. -Prior to using `--sync-to-dashboard`, the target Google Sheet must be set up. See the [readme](gsheet_webapp/README.md) in `gsheet_webapp`. Once that is done, run: +Prior to using `--sync-to-dashboard`, the target Google Sheet must be set up. See the [readme](gsheet_webapp/README.md) in `gsheet_webapp`. Once that is done, run (for example): ``` python packages/redata_reports/main.py --sync-to-dashboard ``` From 06b98ed837db6783919497bdddc4801d5bba167a Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Thu, 31 Jul 2025 17:54:23 -0400 Subject: [PATCH 18/20] Add additional setup for Google Sheet for curators report --- gsheet_webapp/README.md | 24 ++++++++++++- gsheet_webapp/Unpivot.gs | 73 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 gsheet_webapp/Unpivot.gs diff --git a/gsheet_webapp/README.md b/gsheet_webapp/README.md index 2d096b6..424facc 100644 --- a/gsheet_webapp/README.md +++ b/gsheet_webapp/README.md @@ -10,6 +10,8 @@ This Sheet accepts data from the report generator via a POST and stores it for u 1. `items`. Create columns exactly matching the name and order of the keys in the `articles` dictionary from [`items_report.py`](../packages/redata_reports/run/items_report.py) 1. `users`. Create columns exactly matching the name and order of the keys in the `account_info` dictionary from [`userquota_report.py`](../packages/redata_reports/run/userquota_report.py) 1. `curators`. Create columns exactly matching the name and order of the `curators` dictionary from [`curators_report.py`](../packages/trello_reports/run/curators_report.py) + 1. `curators-timeunpivot`. See [Data Transformation Setup](#data-transformation). + 1. `curators-itemunpivot`. See [Data Transformation Setup](#data-transformation). ## WebApp @@ -52,4 +54,24 @@ To deploy the update and keep the app URL the same: 1. Deploy -> Manage deployments -> pencil icon -> Version dropdown -> New Version -> Deploy -Note: a new version is always required, you can't update an existing version. \ No newline at end of file +Note: a new version is always required, you can't update an existing version. + +# Data Transformation + +Additional data transformation is required for some datasets for use in LookerStudio. Namely, the data from the `curators` sheet. + +## Setup + +Add the data manipulation script +1. Go to Extensions -> Apps Script +1. Create a new file `Unpivot.gs` and copy the contents of the corresponding file in this repo to it. + +Create the following formulas +1. `curators-timeunpivot`. In cell A1 enter the following formula +``` +=unpivot({curators!A1:A30,curators!D1:D30,curators!F1:F30,curators!H1:H30,curators!J1:J30,curators!L1:L30,curators!N1:N30,curators!P1:P30,curators!R1:R30,curators!T1:T30,curators!V1:V30,curators!X1:X30,curators!Z1:Z30,curators!AB1:AB30,curators!AD1:AD30,curators!AF1:AF30,curators!AH1:AH30,curators!AJ1:AJ30,curators!AL1:AL30,curators!AN1:AN30,curators!AP1:AP30,curators!AR1:AR30,curators!AT1:AT30,curators!AV1:AV30,curators!AX1:AX30,curators!AZ1:AZ30,curators!BB1:BB30,curators!BD1:BD30,curators!BF1:BF30,curators!BH1:BH30,curators!BJ1:BJ30,curators!BL1:BL30,curators!BN1:BN30,curators!BP1:BP30,curators!BR1:BR30,curators!BT1:BT30,curators!BV1:BV30,curators!BX1:BX30,curators!BZ1:BZ30,curators!CB1:CB30,curators!CD1:CD30,curators!CF1:CF30,curators!CH1:CH30,curators!CJ1:CJ30,curators!CL1:CL30,curators!CN1:CN30,curators!CP1:CP30,curators!CR1:CR30,curators!CT1:CT30,curators!CV1:CV30,curators!CX1:CX30,curators!CZ1:CZ30,curators!DB1:DB30,curators!DD1:DD30,curators!DF1:DF30},1,1,"time_category","time") +``` +1. `curators-itemunpivot`. In cell A1 enter the following formula +``` +=unpivot({curators!A1:A30,curators!C1:C30,curators!E1:E30,curators!G1:G30,curators!I1:I30,curators!K1:K30,curators!M1:M30,curators!O1:O30,curators!Q1:Q30,curators!S1:S30,curators!U1:U30,curators!W1:W30,curators!Y1:Y30,curators!AA1:AA30,curators!AC1:AC30,curators!AE1:AE30,curators!AG1:AG30,curators!AI1:AI30,curators!AK1:AK30,curators!AM1:AM30,curators!AO1:AO30,curators!AQ1:AQ30,curators!AS1:AS30,curators!AU1:AU30,curators!AW1:AW30,curators!AY1:AY30,curators!BA1:BA30,curators!BC1:BC30,curators!BE1:BE30,curators!BG1:BG30,curators!BI1:BI30,curators!BK1:BK30,curators!BM1:BM30,curators!BO1:BO30,curators!BQ1:BQ30,curators!BS1:BS30,curators!BU1:BU30,curators!BW1:BW30,curators!BY1:BY30,curators!CA1:CA30,curators!CC1:CC30,curators!CE1:CE30,curators!CG1:CG30,curators!CI1:CI30,curators!CK1:CK30,curators!CM1:CM30,curators!CO1:CO30,curators!CQ1:CQ30,curators!CS1:CS30,curators!CU1:CU30,curators!CW1:CW30,curators!CY1:CY30,curators!DA1:DA30,curators!DC1:DC30,curators!DE1:DE30},1,1,"item_category","count") +``` \ No newline at end of file diff --git a/gsheet_webapp/Unpivot.gs b/gsheet_webapp/Unpivot.gs new file mode 100644 index 0000000..c2fe181 --- /dev/null +++ b/gsheet_webapp/Unpivot.gs @@ -0,0 +1,73 @@ +/** + * Unpivot a pivot table of any size. + * + * @param {A1:D30} data The pivot table. + * @param {1} fixColumns Number of columns, after which pivoted values begin. Default 1. + * @param {1} fixRows Number of rows (1 or 2), after which pivoted values begin. Default 1. + * @param {"city"} titlePivot The title of horizontal pivot values. Default "column". + * @param {"distance"[,...]} titleValue The title of pivot table values. Default "value". + * @return The unpivoted table + * @customfunction + */ +function unpivot(data,fixColumns,fixRows,titlePivot,titleValue) { + // source: https://stackoverflow.com/a/43681525 + + var fixColumns = fixColumns || 1; // how many columns are fixed + var fixRows = fixRows || 1; // how many rows are fixed + var titlePivot = titlePivot || 'column'; + var titleValue = titleValue || 'value'; + var ret=[],i,j,row,uniqueCols=1; + + // we handle only 2 dimension arrays + if (!Array.isArray(data) || data.length < fixRows || !Array.isArray(data[0]) || data[0].length < fixColumns) + throw new Error('no data'); + // we handle max 2 fixed rows + if (fixRows > 2) + throw new Error('max 2 fixed rows are allowed'); + + // fill empty cells in the first row with value set last in previous columns (for 2 fixed rows) + var tmp = ''; + for (j=0;j 0; i++) + { + // skip totally empty or only whitespace containing rows + if (data[i].join('').replace(/\s+/g,'').length == 0 ) continue; + + // unpivot the row + row = []; + for (j=0;j Date: Thu, 31 Jul 2025 17:58:47 -0400 Subject: [PATCH 19/20] Small fix to gsheet_webapp readme --- gsheet_webapp/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gsheet_webapp/README.md b/gsheet_webapp/README.md index 424facc..c3b6644 100644 --- a/gsheet_webapp/README.md +++ b/gsheet_webapp/README.md @@ -67,11 +67,12 @@ Add the data manipulation script 1. Create a new file `Unpivot.gs` and copy the contents of the corresponding file in this repo to it. Create the following formulas -1. `curators-timeunpivot`. In cell A1 enter the following formula +In `curators-timeunpivot`, in cell A1 enter, the following formula ``` =unpivot({curators!A1:A30,curators!D1:D30,curators!F1:F30,curators!H1:H30,curators!J1:J30,curators!L1:L30,curators!N1:N30,curators!P1:P30,curators!R1:R30,curators!T1:T30,curators!V1:V30,curators!X1:X30,curators!Z1:Z30,curators!AB1:AB30,curators!AD1:AD30,curators!AF1:AF30,curators!AH1:AH30,curators!AJ1:AJ30,curators!AL1:AL30,curators!AN1:AN30,curators!AP1:AP30,curators!AR1:AR30,curators!AT1:AT30,curators!AV1:AV30,curators!AX1:AX30,curators!AZ1:AZ30,curators!BB1:BB30,curators!BD1:BD30,curators!BF1:BF30,curators!BH1:BH30,curators!BJ1:BJ30,curators!BL1:BL30,curators!BN1:BN30,curators!BP1:BP30,curators!BR1:BR30,curators!BT1:BT30,curators!BV1:BV30,curators!BX1:BX30,curators!BZ1:BZ30,curators!CB1:CB30,curators!CD1:CD30,curators!CF1:CF30,curators!CH1:CH30,curators!CJ1:CJ30,curators!CL1:CL30,curators!CN1:CN30,curators!CP1:CP30,curators!CR1:CR30,curators!CT1:CT30,curators!CV1:CV30,curators!CX1:CX30,curators!CZ1:CZ30,curators!DB1:DB30,curators!DD1:DD30,curators!DF1:DF30},1,1,"time_category","time") ``` -1. `curators-itemunpivot`. In cell A1 enter the following formula + +In `curators-itemunpivot` in cell A1, enter the following formula ``` =unpivot({curators!A1:A30,curators!C1:C30,curators!E1:E30,curators!G1:G30,curators!I1:I30,curators!K1:K30,curators!M1:M30,curators!O1:O30,curators!Q1:Q30,curators!S1:S30,curators!U1:U30,curators!W1:W30,curators!Y1:Y30,curators!AA1:AA30,curators!AC1:AC30,curators!AE1:AE30,curators!AG1:AG30,curators!AI1:AI30,curators!AK1:AK30,curators!AM1:AM30,curators!AO1:AO30,curators!AQ1:AQ30,curators!AS1:AS30,curators!AU1:AU30,curators!AW1:AW30,curators!AY1:AY30,curators!BA1:BA30,curators!BC1:BC30,curators!BE1:BE30,curators!BG1:BG30,curators!BI1:BI30,curators!BK1:BK30,curators!BM1:BM30,curators!BO1:BO30,curators!BQ1:BQ30,curators!BS1:BS30,curators!BU1:BU30,curators!BW1:BW30,curators!BY1:BY30,curators!CA1:CA30,curators!CC1:CC30,curators!CE1:CE30,curators!CG1:CG30,curators!CI1:CI30,curators!CK1:CK30,curators!CM1:CM30,curators!CO1:CO30,curators!CQ1:CQ30,curators!CS1:CS30,curators!CU1:CU30,curators!CW1:CW30,curators!CY1:CY30,curators!DA1:DA30,curators!DC1:DC30,curators!DE1:DE30},1,1,"item_category","count") ``` \ No newline at end of file From 74f5e0d39d15bbcfbcc22f4fb3879d0eb876108c Mon Sep 17 00:00:00 2001 From: Fernando Rios Date: Wed, 27 Aug 2025 15:34:27 -0400 Subject: [PATCH 20/20] Update webapp version, add comment for sys.path.insert --- gsheet_webapp/Code.gs | 2 +- packages/redata_reports/run/main.py | 2 +- packages/trello_reports/run/main.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gsheet_webapp/Code.gs b/gsheet_webapp/Code.gs index bf3231c..a2eef73 100644 --- a/gsheet_webapp/Code.gs +++ b/gsheet_webapp/Code.gs @@ -16,7 +16,7 @@ var SCRIPT_PROP = PropertiesService.getScriptProperties(); // new property service -version="1.1.2"; //change when the version changes +version="2.0.0"; //change when the version changes //Best practice for using BetterLog and logging to a spreadsheet: // You can add and set the property "BetterLogLevel" in File > Project Properties and change it to diff --git a/packages/redata_reports/run/main.py b/packages/redata_reports/run/main.py index d3cb37a..536a858 100644 --- a/packages/redata_reports/run/main.py +++ b/packages/redata_reports/run/main.py @@ -11,7 +11,7 @@ import items_report import userquota_report -sys.path.insert(0, 'lib/') +sys.path.insert(0, 'lib/') # Allows loading of shared functions when running locally. No effect when running as DO func. import functions as f import secrets diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py index ab007f3..2712b42 100644 --- a/packages/trello_reports/run/main.py +++ b/packages/trello_reports/run/main.py @@ -9,7 +9,7 @@ from os import environ from version import __version__, __commit__ -sys.path.insert(0, 'lib/') +sys.path.insert(0, 'lib/') # Allows loading of shared functions when running locally. No effect when running as DO func. import curator_report import functions as f import secrets