From 316ea49d1a90e9bdee8db9162af84995977db91d Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Tue, 10 Mar 2026 09:25:19 -0400 Subject: [PATCH 01/27] Issue #153: Reviewed database operations and addressed identified issues --- src/soa_builder/web/app.py | 17 ++--- src/soa_builder/web/audit.py | 69 ------------------ src/soa_builder/web/db.py | 8 ++- src/soa_builder/web/migrate_database.py | 29 ++++++++ src/soa_builder/web/routers/activities.py | 61 ++++++++++------ src/soa_builder/web/routers/freezes.py | 7 +- src/soa_builder/web/routers/rollback.py | 1 - src/soa_builder/web/routers/visits.py | 6 +- src/usdm/generate_activities.py | 13 +++- src/usdm/generate_encounters.py | 85 ++++++++++++++++------- tests/test_routers_activities.py | 6 +- 11 files changed, 155 insertions(+), 147 deletions(-) diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index 01b73ea4..d01f9e09 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -69,6 +69,7 @@ _migrate_study_cell_add_order_index, _migrate_biomedical_concept_audit, _migrate_backfill_biomedical_concept_codes, + _migrate_add_soa_id_indexes, ) from .routers import activities as activities_router from .routers import arms as arms_router @@ -199,6 +200,7 @@ def _configure_logging(): _backfill_dataset_date("protocol_terminology", "protocol_terminology_audit") _migrate_biomedical_concept_audit() _migrate_backfill_biomedical_concept_codes() +_migrate_add_soa_id_indexes() # Include routers @@ -4037,7 +4039,7 @@ def import_matrix(soa_id: int, payload: MatrixImport): next_order += 1 if has_activity_uid: cols.append("activity_uid") - vals.append(f"Activity_{soa_id}_{next_order}") + vals.append(activities_router._next_activity_uid(cur, soa_id)) cur.execute( f"INSERT INTO activity ({','.join(cols)}) VALUES ({','.join(['?'] * len(vals))})", vals, @@ -4076,17 +4078,6 @@ def _reindex(table: str, soa_id: int): ids = [r[0] for r in cur.fetchall()] for idx, _id in enumerate(ids, start=1): cur.execute(f"UPDATE {table} SET order_index=? WHERE id=?", (idx, _id)) - # Maintain activity_uid after any activity reindex - if table == "activity": - # Two-phase UID refresh to satisfy UNIQUE(soa_id, activity_uid) without transient collisions - cur.execute( - "UPDATE activity SET activity_uid = 'TMP_' || id WHERE soa_id=?", - (soa_id,), - ) - cur.execute( - "UPDATE activity SET activity_uid = 'Activity_' || order_index WHERE soa_id=?", - (soa_id,), - ) conn.commit() conn.close() @@ -4179,7 +4170,7 @@ def ui_add_activity(request: Request, soa_id: int, name: str = Form(...)): order_index = cur.fetchone()[0] + 1 cur.execute( "INSERT INTO activity (soa_id,name,order_index,activity_uid) VALUES (?,?,?,?)", - (soa_id, nm, order_index, f"Activity_{order_index}"), + (soa_id, nm, order_index, activities_router._next_activity_uid(cur, soa_id)), ) aid = cur.lastrowid conn.commit() diff --git a/src/soa_builder/web/audit.py b/src/soa_builder/web/audit.py index e0835f6c..b66ccadc 100644 --- a/src/soa_builder/web/audit.py +++ b/src/soa_builder/web/audit.py @@ -45,18 +45,6 @@ def _record_element_audit( try: conn = _connect() cur = conn.cursor() - # Ensure table exists (defensive for migrated databases) - cur.execute( - """CREATE TABLE IF NOT EXISTS element_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - soa_id INTEGER NOT NULL, - element_id INTEGER, - action TEXT NOT NULL, - before_json TEXT, - after_json TEXT, - performed_at TEXT NOT NULL - )""" - ) cur.execute( "INSERT INTO element_audit (soa_id, element_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", ( @@ -163,18 +151,6 @@ def _record_study_cell_audit( try: conn = _connect() cur = conn.cursor() - # Ensure table exists (defensive for migrated databases) - cur.execute( - """CREATE TABLE IF NOT EXISTS study_cell_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - soa_id INTEGER NOT NULL, - study_cell_id INTEGER, - action TEXT NOT NULL, - before_json TEXT, - after_json TEXT, - performed_at TEXT NOT NULL - )""" - ) cur.execute( "INSERT INTO study_cell_audit (soa_id, study_cell_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", ( @@ -256,18 +232,6 @@ def _record_instance_audit( try: conn = _connect() cur = conn.cursor() - # Ensure table exists defensively - cur.execute( - """CREATE TABLE IF NOT EXISTS instance_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - soa_id INTEGER NOT NULL, - instance_id INTEGER, - action TEXT NOT NULL, - before_json TEXT, - after_json TEXT, - performed_at TEXT NOT NULL - )""" - ) cur.execute( "INSERT INTO instance_audit (soa_id, instance_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", ( @@ -295,17 +259,6 @@ def _record_decision_instance_audit( try: conn = _connect() cur = conn.cursor() - cur.execute( - """CREATE TABLE IF NOT EXISTS decision_instance_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - soa_id INTEGER NOT NULL, - decision_instance_id INTEGER, - action TEXT NOT NULL, - before_json TEXT, - after_json TEXT, - performed_at TEXT NOT NULL - )""" - ) cur.execute( "INSERT INTO decision_instance_audit (soa_id, decision_instance_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", ( @@ -333,17 +286,6 @@ def _record_condition_assignment_audit( try: conn = _connect() cur = conn.cursor() - cur.execute( - """CREATE TABLE IF NOT EXISTS condition_assignment_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - soa_id INTEGER NOT NULL, - condition_assignment_id INTEGER, - action TEXT NOT NULL, - before_json TEXT, - after_json TEXT, - performed_at TEXT NOT NULL - )""" - ) cur.execute( "INSERT INTO condition_assignment_audit (soa_id, condition_assignment_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", ( @@ -403,17 +345,6 @@ def _record_biomedical_concept_audit( if own_conn: conn = _connect() cur = conn.cursor() - cur.execute( - """CREATE TABLE IF NOT EXISTS biomedical_concept_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - soa_id INTEGER NOT NULL, - biomedical_concept_id INTEGER, - action TEXT NOT NULL, - before_json TEXT, - after_json TEXT, - performed_at TEXT NOT NULL - )""" - ) cur.execute( "INSERT INTO biomedical_concept_audit" " (soa_id, biomedical_concept_id, action, before_json, after_json, performed_at)" diff --git a/src/soa_builder/web/db.py b/src/soa_builder/web/db.py index 17fc22c3..6061237f 100644 --- a/src/soa_builder/web/db.py +++ b/src/soa_builder/web/db.py @@ -48,6 +48,10 @@ def _connect(): conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA synchronous=NORMAL") conn.execute("PRAGMA busy_timeout=3000") - except Exception: - pass + except Exception as e: + import logging + + logging.getLogger("soa_builder.db").warning( + "PRAGMA configuration failed on %s: %s", db_path, e + ) return conn diff --git a/src/soa_builder/web/migrate_database.py b/src/soa_builder/web/migrate_database.py index 71c678ed..ccefd840 100644 --- a/src/soa_builder/web/migrate_database.py +++ b/src/soa_builder/web/migrate_database.py @@ -1122,3 +1122,32 @@ def _migrate_biomedical_concept_property_add_uid(): conn.close() except Exception as e: logger.warning("_migrate_biomedical_concept_property_add_uid: %s", e) + + +def _migrate_add_soa_id_indexes(): + """Add standalone soa_id indexes on high-traffic tables. + + The existing UNIQUE constraints cover (soa_id, uid) lookups, but bare + WHERE soa_id=? list queries do full table scans without a leading index. + These indexes cover the ~259 soa_id filter sites in the codebase. + """ + try: + conn = _connect() + cur = conn.cursor() + indexes = [ + ("idx_activity_soa", "activity", "soa_id"), + ("idx_visit_soa", "visit", "soa_id"), + ("idx_matrix_cells_soa", "matrix_cells", "soa_id"), + ("idx_activity_concept_soa", "activity_concept", "soa_id"), + ("idx_instances_soa", "instances", "soa_id"), + ("idx_timing_soa", "timing", "soa_id"), + ] + created = [] + for idx_name, table, col in indexes: + cur.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({col})") + created.append(idx_name) + conn.commit() + conn.close() + logger.info("_migrate_add_soa_id_indexes: ensured indexes %s", created) + except Exception as e: + logger.warning("_migrate_add_soa_id_indexes: %s", e) diff --git a/src/soa_builder/web/routers/activities.py b/src/soa_builder/web/routers/activities.py index 48d675ce..4603da5f 100644 --- a/src/soa_builder/web/routers/activities.py +++ b/src/soa_builder/web/routers/activities.py @@ -6,7 +6,7 @@ import time from typing import List -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, Form +from fastapi import APIRouter, BackgroundTasks, Body, HTTPException, Request, Form from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates @@ -137,6 +137,41 @@ def get_activity(soa_id: int, activity_id: int): } +def _next_activity_uid(cur, soa_id: int) -> str: + """Return the next Activity_N UID, never reusing a deleted one. + + Scans both the live table and the audit trail so deleted UIDs are + never recycled — matching the pattern used by _next_study_cell_uid. + """ + max_n = 0 + cur.execute("SELECT activity_uid FROM activity WHERE soa_id=?", (soa_id,)) + for (uid,) in cur.fetchall(): + if isinstance(uid, str) and uid.startswith("Activity_"): + try: + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except (ValueError, IndexError): + pass + cur.execute( + "SELECT before_json, after_json FROM activity_audit WHERE soa_id=?", + (soa_id,), + ) + for before_raw, after_raw in cur.fetchall(): + for raw in (before_raw, after_raw): + if not raw: + continue + try: + uid = json.loads(raw).get("activity_uid", "") + if isinstance(uid, str) and uid.startswith("Activity_"): + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except Exception: + pass + return f"Activity_{max_n + 1}" + + @router.post("/activities", response_class=JSONResponse) def add_activity(soa_id: int, payload: ActivityCreate): if not soa_exists(soa_id): @@ -148,8 +183,7 @@ def add_activity(soa_id: int, payload: ActivityCreate): "SELECT COALESCE(MAX(order_index),0) FROM activity WHERE soa_id=?", (soa_id,) ) order_index = (cur.fetchone() or [0])[0] + 1 - # Compute activity_uid from order_index (keeps list stable after inserts) - activity_uid = f"Activity_{order_index}" + activity_uid = _next_activity_uid(cur, soa_id) name = (payload.name or "").strip() label = (payload.label or "").strip() or None @@ -313,7 +347,7 @@ def ui_update_activity( @router.post("/activities/reorder", response_class=JSONResponse) -def reorder_activities_api(soa_id: int, order: List[int]): +def reorder_activities_api(soa_id: int, order: List[int] = Body(..., embed=True)): if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: @@ -355,14 +389,6 @@ def reorder_activities_api(soa_id: int, order: List[int]): ).fetchall() } - # Reassign activity_uid from order_index - cur.execute( - "UPDATE activity SET activity_uid='TMP_' || id WHERE soa_id=?", (soa_id,) - ) - cur.execute( - "UPDATE activity SET activity_uid='Activity_' || order_index WHERE soa_id=?", - (soa_id,), - ) conn.commit() conn.close() @@ -411,7 +437,7 @@ def add_activities_bulk(soa_id: int, payload: BulkActivities): order_index += 1 cur.execute( "INSERT INTO activity (soa_id,name,order_index,activity_uid) VALUES (?,?,?,?)", - (soa_id, name, order_index, f"Activity_{order_index}"), + (soa_id, name, order_index, _next_activity_uid(cur, soa_id)), ) added.append(name) existing.add(lname) @@ -554,7 +580,7 @@ def set_activity_concepts( def _reindex_activities(soa_id: int): - """Re-number order_index and activity_uid after a delete.""" + """Re-number order_index after a delete. activity_uid is immutable and never changed.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -563,13 +589,6 @@ def _reindex_activities(soa_id: int): ids = [r[0] for r in cur.fetchall()] for idx, _id in enumerate(ids, start=1): cur.execute("UPDATE activity SET order_index=? WHERE id=?", (idx, _id)) - cur.execute( - "UPDATE activity SET activity_uid = 'TMP_' || id WHERE soa_id=?", (soa_id,) - ) - cur.execute( - "UPDATE activity SET activity_uid = 'Activity_' || order_index WHERE soa_id=?", - (soa_id,), - ) conn.commit() conn.close() diff --git a/src/soa_builder/web/routers/freezes.py b/src/soa_builder/web/routers/freezes.py index 839738e2..62704520 100644 --- a/src/soa_builder/web/routers/freezes.py +++ b/src/soa_builder/web/routers/freezes.py @@ -1,14 +1,13 @@ import json import logging import os -import sqlite3 from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, JSONResponse from fastapi.templating import Jinja2Templates +from ..db import _connect from ..utils import soa_exists -DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "templates") templates = Jinja2Templates(directory=TEMPLATES_DIR) @@ -16,10 +15,6 @@ logger = logging.getLogger("soa_builder.web.routers.freezes") -def _connect(): - return sqlite3.connect(DB_PATH) - - # Removed local _soa_exists; using shared utils.soa_exists diff --git a/src/soa_builder/web/routers/rollback.py b/src/soa_builder/web/routers/rollback.py index 01a3d509..1a6f66a5 100644 --- a/src/soa_builder/web/routers/rollback.py +++ b/src/soa_builder/web/routers/rollback.py @@ -9,7 +9,6 @@ from ..utils import soa_exists -DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "templates") templates = Jinja2Templates(directory=TEMPLATES_DIR) diff --git a/src/soa_builder/web/routers/visits.py b/src/soa_builder/web/routers/visits.py index f40dcd21..aa4f059e 100644 --- a/src/soa_builder/web/routers/visits.py +++ b/src/soa_builder/web/routers/visits.py @@ -457,7 +457,6 @@ def update_visit(soa_id: int, visit_id: int, payload: VisitUpdate): soa_id, ), ) - conn.commit() if new_environmental_value is not None: if not env_code_uid: @@ -498,8 +497,6 @@ def update_visit(soa_id: int, visit_id: int, payload: VisitUpdate): (env_code_uid, visit_id, soa_id), ) - conn.commit() - if new_contact_mode is not None: if not contact_mode_code_uid: contact_mode_code_uid = _get_next_code_uid(cur, soa_id) @@ -539,7 +536,7 @@ def update_visit(soa_id: int, visit_id: int, payload: VisitUpdate): (contact_mode_code_uid, visit_id, soa_id), ) - conn.commit() + conn.commit() cur.execute( """ @@ -643,6 +640,7 @@ def delete_visit(soa_id: int, visit_id: int): ) row = cur.fetchone() if not row: + conn.close() raise HTTPException(404, f"Encounter id={int(visit_id)} not found") before = { diff --git a/src/usdm/generate_activities.py b/src/usdm/generate_activities.py index 16d2d6de..e1620f13 100755 --- a/src/usdm/generate_activities.py +++ b/src/usdm/generate_activities.py @@ -2,7 +2,6 @@ from typing import List, Dict, Any from soa_builder.web.db import _connect from soa_builder.web.utils import _nz -from .usdm_utils import _get_biomedical_concept_ids def build_usdm_activities(soa_id: int) -> List[Dict[str, Any]]: @@ -42,10 +41,18 @@ def build_usdm_activities(soa_id: int) -> List[Dict[str, Any]]: (soa_id,), ) rows = cur.fetchall() + + # Pre-fetch all concept mappings for this SOA in one query to avoid N+1 + cur.execute( + "SELECT activity_uid, concept_uid FROM activity_concept WHERE soa_id=?", + (soa_id,), + ) + bc_map: dict[str, list[str]] = {} + for act_uid, concept_uid in cur.fetchall(): + bc_map.setdefault(act_uid, []).append(concept_uid) conn.close() # Build simple linear previous/next links by list order - # ids = [f"Activity_{r[0]}" for r in rows] uids = [r[1] for r in rows] id_by_index = {i: uid for i, uid in enumerate(uids)} @@ -55,7 +62,7 @@ def build_usdm_activities(soa_id: int) -> List[Dict[str, Any]]: aid = activity_uid prev_id = id_by_index.get(i - 1) next_id = id_by_index.get(i + 1) - bcs = _get_biomedical_concept_ids(soa_id, aid) + bcs = bc_map.get(aid, []) activity = { "id": aid, diff --git a/src/usdm/generate_encounters.py b/src/usdm/generate_encounters.py index e2488b7b..6a596ac7 100644 --- a/src/usdm/generate_encounters.py +++ b/src/usdm/generate_encounters.py @@ -2,12 +2,6 @@ from typing import List, Dict, Any, Tuple from soa_builder.web.utils import get_submission_value_for_code, _nz from soa_builder.web.db import _connect -from .usdm_utils import ( - _get_timing_name, - _get_transition_start_rule, - _get_transition_end_rule, - _get_code_tuple, -) # Override the definition in usdm_utils.py @@ -81,6 +75,54 @@ def build_usdm_encounters(soa_id: int) -> List[Dict[str, Any]]: (soa_id,), ) rows = cur.fetchall() + + # Pre-fetch code_association + ddf_terminology for type codes (keyed by code_uid) + cur.execute( + "SELECT DISTINCT c.code_uid, c.codelist_table, p.code, p.cdisc_submission_value, p.dataset_date " + "FROM code_association c INNER JOIN ddf_terminology p ON c.codelist_code = p.codelist_code " + "AND c.code = p.code WHERE c.soa_id=?", + (soa_id,), + ) + type_code_map: dict = {} + for code_uid, codelist_table, code, decode, dataset_date in cur.fetchall(): + type_code_map.setdefault(code_uid, ([], [], [], [])) + type_code_map[code_uid][0].append(code) + type_code_map[code_uid][1].append(decode) + type_code_map[code_uid][2].append(codelist_table) + type_code_map[code_uid][3].append(dataset_date) + + # Pre-fetch code_association for env/contact codes (keyed by code_uid) + cur.execute( + "SELECT DISTINCT code_uid, codelist_table, code FROM code_association WHERE soa_id=?", + (soa_id,), + ) + code_tuple_map: dict = {} + for code_uid, codelist_table, code in cur.fetchall(): + code_tuple_map.setdefault(code_uid, ([], [])) + code_tuple_map[code_uid][0].append(code) + code_tuple_map[code_uid][1].append(codelist_table) + + # Pre-fetch all transition rules for this SOA (keyed by transition_rule_uid) + cur.execute( + "SELECT transition_rule_uid, name, label, description, text FROM transition_rule WHERE soa_id=?", + (soa_id,), + ) + transition_rule_map: dict = {} + for tr_uid, tr_name, tr_label, tr_desc, tr_text in cur.fetchall(): + transition_rule_map[tr_uid] = { + "id": tr_uid, + "extensionAttributes": [], + "name": tr_name or None, + "label": tr_label or None, + "description": tr_desc or None, + "text": tr_text or None, + "instanceType": "TransitionRule", + } + + # Pre-fetch all timing UIDs for this SOA (keyed by timing id) + cur.execute("SELECT id, timing_uid FROM timing WHERE soa_id=?", (soa_id,)) + timing_id_map: dict = {row[0]: row[1] for row in cur.fetchall()} + conn.close() uids = [r[3] for r in rows] @@ -116,42 +158,33 @@ def build_usdm_encounters(soa_id: int) -> List[Dict[str, Any]]: r[10], ) eid = encounter_uid - t_code, t_decode, t_codeSystem, t_codeSystemVersion = _get_type_code_tuple( - soa_id, type - ) + _type_entry = type_code_map.get(type, ([], [], [], [])) + t_code, t_decode, t_codeSystem, t_codeSystemVersion = _type_entry e_code: List[str] = [] e_codesystem: List[str] = [] if environmentalSettings: - e_code, e_codesystem = _get_code_tuple(soa_id, environmentalSettings) + e_code, e_codesystem = code_tuple_map.get(environmentalSettings, ([], [])) c_code: List[str] = [] c_codesystem: List[str] = [] if contactModes: - c_code, c_codesystem = _get_code_tuple(soa_id, contactModes) + c_code, c_codesystem = code_tuple_map.get(contactModes, ([], [])) - # print(e_code, e_codesystem) prev_id = id_by_index.get(i - 1) next_id = id_by_index.get(i + 1) - timing_uid = _get_timing_name( - soa_id, - ( - int(scheduledAtId) - if (scheduledAtId is not None and str(scheduledAtId).isdigit()) - else None - ), + _sched_id = ( + int(scheduledAtId) + if (scheduledAtId is not None and str(scheduledAtId).isdigit()) + else None ) + timing_uid = timing_id_map.get(_sched_id) if _sched_id is not None else None - transition_start_rule_obj = _get_transition_start_rule( - soa_id, transition_start_rule_uid - ) - - transition_end_rule_obj = _get_transition_end_rule( - soa_id, transition_end_rule_uid - ) + transition_start_rule_obj = transition_rule_map.get(transition_start_rule_uid) + transition_end_rule_obj = transition_rule_map.get(transition_end_rule_uid) # Build optional environmentalSettings array env_settings: List[Dict[str, Any]] = [] diff --git a/tests/test_routers_activities.py b/tests/test_routers_activities.py index 19147ca0..ac9e9c2a 100644 --- a/tests/test_routers_activities.py +++ b/tests/test_routers_activities.py @@ -178,7 +178,9 @@ def test_reorder_activities(): ] # Reorder - resp = client.post(f"/soa/{soa_id}/activities/reorder", json=[a3, a1, a2]) + resp = client.post( + f"/soa/{soa_id}/activities/reorder", json={"order": [a3, a1, a2]} + ) assert resp.status_code == 200 @@ -196,7 +198,7 @@ def test_reorder_activities_router(): ] # Reorder via router - resp = client.post(f"/soa/{soa_id}/activities/reorder", json=[a2, a1]) + resp = client.post(f"/soa/{soa_id}/activities/reorder", json={"order": [a2, a1]}) assert resp.status_code == 200 From 5ea155691717b45deeb6e471457d30050af98409 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:54:21 -0400 Subject: [PATCH 02/27] Added lines for .scripts directory and interim USDM --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 72f6bf2a..912b269f 100644 --- a/.gitignore +++ b/.gitignore @@ -102,5 +102,7 @@ CLAUDE.md edit-column-collapse.html .claude api_test.py +.scripts +files/pilot_LZZT_narrative_2026MAR10.json # End of file From ece559e0d2e6898b41723e556a3e6313e7aab863 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:54:50 -0400 Subject: [PATCH 03/27] Fixed 404 on activity save --- src/soa_builder/web/templates/activities.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/soa_builder/web/templates/activities.html b/src/soa_builder/web/templates/activities.html index 98ef8d11..13dd5085 100644 --- a/src/soa_builder/web/templates/activities.html +++ b/src/soa_builder/web/templates/activities.html @@ -61,7 +61,7 @@

Activities for Study: {% if study_label %}{{ study_label }}{% else %}{{ stud {% for a in activities %} -
+ {{ a.activity_uid }} From 95ee7967f6cbc6b712ce2b58c8028e468e80b001 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:05:47 -0400 Subject: [PATCH 04/27] Issue #164: Added new footnotes feature --- files/NCT01797120_Footnote_1.html | 79 +++++++ files/NCT01797120_Footnote_2.html | 17 ++ src/soa_builder/web/app.py | 30 +++ src/soa_builder/web/audit.py | 27 +++ src/soa_builder/web/initialize_database.py | 15 ++ src/soa_builder/web/migrate_database.py | 48 ++++ src/soa_builder/web/routers/footnotes.py | 257 +++++++++++++++++++++ src/soa_builder/web/templates/edit.html | 57 +++++ 8 files changed, 530 insertions(+) create mode 100644 files/NCT01797120_Footnote_1.html create mode 100644 files/NCT01797120_Footnote_2.html create mode 100644 src/soa_builder/web/routers/footnotes.py diff --git a/files/NCT01797120_Footnote_1.html b/files/NCT01797120_Footnote_1.html new file mode 100644 index 00000000..a98d7544 --- /dev/null +++ b/files/NCT01797120_Footnote_1.html @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
a:≤ 4 weeks of randomization; if assessments required ≤ 7 days of Cycle 1 Day 1 (C1D1), they do not need to be repeated (includes labs).
b:≤ 7 days prior to the start of C1D1.
c:+/- 72 hour window allowed prior to D1 of each subsequent cycle after the first cycle for scheduled therapy/tests/visits. Delay due to holidays, weekends, +bad weather or other unforeseen circumstances will be permitted.
d:In the event of grade 3 or 4 hematologic toxicity, CBC with differential and platelet count will be obtained every 1-3 days until there is evidence of +hematologic recovery.
e:+/- 72 hours prior to Cycle 2 Day 1 then approximately every 12 weeks during treatment (and more frequently as clinically indicated), and at end of +treatment.
f:All patients should be screened for hepatitis risk factors and any past illnesses of hepatitis B and hepatitis C infection (see Section 7.1.1). All patients with a +positive medical history per Section 7.1.1 need hepatitis testing as noted on above table. It is highly recommended that patients positive for HBV-DNA or +HBsAg are treated prophylactically with an antiviral (e.g., Lamivudine) for 1-2 weeks prior to receiving study drug (see Table 5-3). The antiviral treatment +should continue throughout the entire study period and for at least 4 weeks after the last dose of everolimus. +
g:Patients on antiviral prophylaxis treatment or positive HBV antibodies should be tested for HBV-DNA ≤ 7 days prior to the start of C1D1 and +/- 72 hrs prior +to D1 of each subsequent cycle to monitor for reactivation. See Table 5-4 for reactivation instructions.
h:Patients with positive HCV RNA-PCR results at screening and/or a history of past infection (even if treated and considered ‘cured’) should have HCV RNA- +PCR testing performed on ≤ 7 days prior to the start of C1D1 and +/- 72 hrs prior to D1 of each subsequent cycle to monitor for flare. Everolimus must be +discontinued if HCV flare is confirmed according to the guidance in Table 5-5.
i:Tumor measurements may be made using physical examination, CT Scans or MRI. Tumor assessments will be performed every 12 weeks, +/- 1 week +(every 3 months). Imaging will include chest and abdomen. Bone Scans and Brain CT/MRI may be performed as clinically indicated. Scans do not have to be +repeated once disease progression is documented.
j:Adverse events related to fulvestrant and/or everolimus/placebo will be followed for 30 days after the last dose of study therapy (fulvestrant and/or +everolimus/placebo) or until ≤ grade 1 or if the grade is >1, the event must be permanent and stable. Please note- Serious adverse events >30 days after last +dose of fulvestrant and/or everolimus/placebo are not reported unless the event may be related to everolimus/placebo.
k:CBC and chemistry may be used to assess ongoing toxicity but are not required in the Continuation Phase for patients who receive fulvestrant alone. +Patients who continue fulvestrant with everolimus should periodically have CBC, chemistries, fasting glucose, fasting lipids, HBV DNA, HCV RNA-PCR per +labeling guidelines.
l:Study Drug Compliance (Pill Diary) for those patients who receive everolimus in the Continuation Phase.
m:All patients including those that discontinue protocol therapy will be followed for 3 years from the time of randomization. Patients that have not progressed +during the Induction or Continuation Phase will continue to have imaging scans completed every 12 weeks, +/- 1 week (every 3 months) until documented +progression.
n:PFTs with DLCO as medically indicated only, (PFTs are not otherwise required during course of study).
o:Follow every 3 months for disease progression and survival. Initiation of any new systemic therapy will also be documented
\ No newline at end of file diff --git a/files/NCT01797120_Footnote_2.html b/files/NCT01797120_Footnote_2.html new file mode 100644 index 00000000..cb3f82cd --- /dev/null +++ b/files/NCT01797120_Footnote_2.html @@ -0,0 +1,17 @@ + + + + + + + + + + + + + +
*Cycle 1, Day 1 is defined as the first day on which fulvestrant is given in combination with placebo/everolimus (the second fulvestrant dose is given on day 15 +of the first cycle only). Day 1 of each additional cycle is defined as the day in which fulvestrant is given in combination with everolimus/placebo.
^End of Induction/End of Treatment should be performed within 30 days of last dose of fulvestrant.
£Continuation Phase: Patients in the Continuation Phase should continue to receive fulvestrant alone (if originally randomized to placebo) or in combination +with everolimus (if originally randomized to everolimus) at the same dose and schedule (+/- 1 week window for scheduled therapy/tests/visits; delays due to +holidays, weekends, bad weather or other unforeseen circumstances will be permitted) until disease progression or unacceptable toxicity.
\ No newline at end of file diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index d01f9e09..aa008550 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -70,6 +70,8 @@ _migrate_biomedical_concept_audit, _migrate_backfill_biomedical_concept_codes, _migrate_add_soa_id_indexes, + _migrate_add_footnote_table, + _migrate_add_footnote_audit_table, ) from .routers import activities as activities_router from .routers import arms as arms_router @@ -88,6 +90,7 @@ from .routers import tdd as tdd_router from .routers import decision_instances as decision_instances_router from .routers import condition_assignments as condition_assignments_router +from .routers import footnotes as footnotes_router from .audit import _record_element_audit @@ -201,6 +204,8 @@ def _configure_logging(): _migrate_biomedical_concept_audit() _migrate_backfill_biomedical_concept_codes() _migrate_add_soa_id_indexes() +_migrate_add_footnote_table() +_migrate_add_footnote_audit_table() # Include routers @@ -222,6 +227,8 @@ def _configure_logging(): app.include_router(tdd_router.router) app.include_router(decision_instances_router.router) app.include_router(condition_assignments_router.router) +app.include_router(footnotes_router.router) +app.include_router(footnotes_router.ui_router) def _record_visit_audit( @@ -4633,6 +4640,28 @@ def ui_edit(request: Request, soa_id: int): if not default_timeline and "unassigned" in instances_by_timeline: default_timeline = "unassigned" + # Load footnotes for display below matrix + conn_fn = _connect() + cur_fn = conn_fn.cursor() + cur_fn.execute( + "SELECT id,soa_id,footnote_uid,name,label,description,text,dictionary_uid FROM footnote WHERE soa_id=? ORDER BY id", + (soa_id,), + ) + footnotes = [ + dict( + id=r[0], + soa_id=r[1], + footnote_uid=r[2], + name=r[3], + label=r[4], + description=r[5], + text=r[6], + dictionary_uid=r[7], + ) + for r in cur_fn.fetchall() + ] + conn_fn.close() + instances_crud = instances_router.list_instances(soa_id) encounter_options = get_encounter_id(soa_id) epoch_options = get_epoch_uid(soa_id) @@ -4676,6 +4705,7 @@ def ui_edit(request: Request, soa_id: int): "timelines": timelines, "instances_by_timeline": instances_by_timeline, "default_timeline": default_timeline, + "footnotes": footnotes, }, ) diff --git a/src/soa_builder/web/audit.py b/src/soa_builder/web/audit.py index b66ccadc..6cecd9e2 100644 --- a/src/soa_builder/web/audit.py +++ b/src/soa_builder/web/audit.py @@ -363,3 +363,30 @@ def _record_biomedical_concept_audit( conn.close() except Exception as e: logger.warning("Failed recording biomedical_concept audit: %s", e) + + +def _record_footnote_audit( + soa_id: int, + action: str, + footnote_id: Optional[int], + before: Optional[Dict[str, Any]] = None, + after: Optional[Dict[str, Any]] = None, +): + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + "INSERT INTO footnote_audit (soa_id, footnote_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + footnote_id, + action, + json.dumps(before) if before else None, + json.dumps(after) if after else None, + datetime.now(timezone.utc).isoformat(), + ), + ) + conn.commit() + conn.close() + except Exception as e: + logger.warning("Failed recording footnote audit: %s", e) diff --git a/src/soa_builder/web/initialize_database.py b/src/soa_builder/web/initialize_database.py index de5ff084..d886d40c 100644 --- a/src/soa_builder/web/initialize_database.py +++ b/src/soa_builder/web/initialize_database.py @@ -352,6 +352,21 @@ def _init_db(): )""" ) + # The footnote table (created until full incorporation of SyntaxTemplates) + cur.execute( + """CREATE TABLE IF NOT EXISTS footnote ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INT, + footnote_uid TEXT NOT NULL, + name TEXT NOT NULL, + label TEXT, + description TEXT, + text TEXT, + dictionary_uid TEXT, + UNIQUE(soa_id, footnote_uid) + )""" + ) + # AUDIT TABLES FOR TRACKING ALL CHANGES TO ENTITIES # Element audit table capturing create/update/delete operations diff --git a/src/soa_builder/web/migrate_database.py b/src/soa_builder/web/migrate_database.py index ccefd840..6f043798 100644 --- a/src/soa_builder/web/migrate_database.py +++ b/src/soa_builder/web/migrate_database.py @@ -1151,3 +1151,51 @@ def _migrate_add_soa_id_indexes(): logger.info("_migrate_add_soa_id_indexes: ensured indexes %s", created) except Exception as e: logger.warning("_migrate_add_soa_id_indexes: %s", e) + + +def _migrate_add_footnote_table(): + """Add the database table footnote""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + """CREATE TABLE IF NOT EXISTS footnote ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INT, + footnote_uid TEXT NOT NULL, + name TEXT NOT NULL, + label TEXT, + description TEXT, + text TEXT, + dictionary_uid TEXT, + UNIQUE(soa_id, footnote_uid) + )""" + ) + conn.commit() + conn.close() + logger.info("_migrate_add_footnote_table created footnote table") + except Exception as e: + logger.warning("_migrate_add_footnote_table failed: %s", e) + + +def _migrate_add_footnote_audit_table(): + """Create footnote_audit table for tracking create/update/delete operations.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + """CREATE TABLE IF NOT EXISTS footnote_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + footnote_id INTEGER, + action TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) + conn.commit() + conn.close() + logger.info("_migrate_add_footnote_audit_table created footnote_audit table") + except Exception as e: + logger.warning("_migrate_add_footnote_audit_table failed: %s", e) diff --git a/src/soa_builder/web/routers/footnotes.py b/src/soa_builder/web/routers/footnotes.py new file mode 100644 index 00000000..0fdb74af --- /dev/null +++ b/src/soa_builder/web/routers/footnotes.py @@ -0,0 +1,257 @@ +import logging + +from fastapi import APIRouter, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse + +from ..audit import _record_footnote_audit +from ..db import _connect +from ..utils import soa_exists + +router = APIRouter(prefix="/soa/{soa_id}") +ui_router = APIRouter() +logger = logging.getLogger("soa_builder.web.routers.footnotes") + + +def _next_footnote_uid(soa_id: int) -> str: + """Return next Footnote_N UID, never reusing deleted UIDs.""" + conn = _connect() + cur = conn.cursor() + cur.execute("SELECT MAX(id) FROM footnote WHERE soa_id=?", (soa_id,)) + row = cur.fetchone() + live_max = row[0] or 0 + cur.execute("SELECT MAX(footnote_id) FROM footnote_audit WHERE soa_id=?", (soa_id,)) + row = cur.fetchone() + audit_max = row[0] or 0 + conn.close() + return f"Footnote_{max(live_max, audit_max) + 1}" + + +def _row_to_dict(row) -> dict: + keys = [ + "id", + "soa_id", + "footnote_uid", + "name", + "label", + "description", + "text", + "dictionary_uid", + ] + return dict(zip(keys, row)) + + +# --------------------------------------------------------------------------- +# JSON API endpoints +# --------------------------------------------------------------------------- + + +@router.get("/footnotes", response_class=JSONResponse) +def list_footnotes(soa_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,footnote_uid,name,label,description,text,dictionary_uid FROM footnote WHERE soa_id=? ORDER BY id", + (soa_id,), + ) + rows = [_row_to_dict(r) for r in cur.fetchall()] + conn.close() + return JSONResponse(rows) + + +@router.post("/footnotes", response_class=JSONResponse) +def create_footnote( + soa_id: int, + name: str = Form(...), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), + dictionary_uid: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + uid = _next_footnote_uid(soa_id) + conn = _connect() + cur = conn.cursor() + cur.execute( + "INSERT INTO footnote (soa_id, footnote_uid, name, label, description, text, dictionary_uid) VALUES (?,?,?,?,?,?,?)", + ( + soa_id, + uid, + name, + label or None, + description or None, + text or None, + dictionary_uid or None, + ), + ) + conn.commit() + footnote_id = cur.lastrowid + after = { + "footnote_uid": uid, + "name": name, + "label": label, + "description": description, + "text": text, + "dictionary_uid": dictionary_uid, + } + conn.close() + _record_footnote_audit(soa_id, "create", footnote_id, before=None, after=after) + return JSONResponse( + {"id": footnote_id, "footnote_uid": uid, **after}, status_code=201 + ) + + +@router.patch("/footnotes/{footnote_id}", response_class=JSONResponse) +def update_footnote( + soa_id: int, + footnote_id: int, + name: str | None = Form(None), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), + dictionary_uid: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,footnote_uid,name,label,description,text,dictionary_uid FROM footnote WHERE id=? AND soa_id=?", + (footnote_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Footnote not found") + before = _row_to_dict(row) + new_name = name if name is not None else before["name"] + new_label = label if label is not None else before["label"] + new_desc = description if description is not None else before["description"] + new_text = text if text is not None else before["text"] + new_dict_uid = ( + dictionary_uid if dictionary_uid is not None else before["dictionary_uid"] + ) + cur.execute( + "UPDATE footnote SET name=?, label=?, description=?, text=?, dictionary_uid=? WHERE id=? AND soa_id=?", + ( + new_name, + new_label or None, + new_desc or None, + new_text or None, + new_dict_uid or None, + footnote_id, + soa_id, + ), + ) + conn.commit() + conn.close() + after = { + **before, + "name": new_name, + "label": new_label, + "description": new_desc, + "text": new_text, + "dictionary_uid": new_dict_uid, + } + _record_footnote_audit(soa_id, "update", footnote_id, before=before, after=after) + return JSONResponse(after) + + +@router.delete("/footnotes/{footnote_id}", response_class=JSONResponse) +def delete_footnote(soa_id: int, footnote_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,footnote_uid,name,label,description,text,dictionary_uid FROM footnote WHERE id=? AND soa_id=?", + (footnote_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Footnote not found") + before = _row_to_dict(row) + cur.execute("DELETE FROM footnote WHERE id=? AND soa_id=?", (footnote_id, soa_id)) + conn.commit() + conn.close() + _record_footnote_audit(soa_id, "delete", footnote_id, before=before, after=None) + return JSONResponse({"deleted": footnote_id}) + + +# --------------------------------------------------------------------------- +# UI form endpoints +# --------------------------------------------------------------------------- + + +@ui_router.post("/ui/soa/{soa_id}/footnotes/create", response_class=HTMLResponse) +def ui_create_footnote( + request: Request, + soa_id: int, + name: str = Form(...), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), + dictionary_uid: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + create_footnote( + soa_id, + name=name, + label=label, + description=description, + text=text, + dictionary_uid=dictionary_uid, + ) + redirect_url = f"/ui/soa/{soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return HTMLResponse(f"") + + +@router.post("/footnotes/{footnote_id}/update", response_class=HTMLResponse) +def ui_update_footnote( + request: Request, + soa_id: int, + footnote_id: int, + name: str | None = Form(None), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), + dictionary_uid: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + update_footnote( + soa_id, + footnote_id, + name=name, + label=label, + description=description, + text=text, + dictionary_uid=dictionary_uid, + ) + redirect_url = f"/ui/soa/{soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return HTMLResponse(f"") + + +@ui_router.post( + "/ui/soa/{soa_id}/footnotes/{footnote_id}/delete", response_class=HTMLResponse +) +def ui_delete_footnote( + request: Request, + soa_id: int, + footnote_id: int, +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + delete_footnote(soa_id, footnote_id) + redirect_url = f"/ui/soa/{soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return HTMLResponse(f"") diff --git a/src/soa_builder/web/templates/edit.html b/src/soa_builder/web/templates/edit.html index ec9f9cc6..b5ac7e81 100644 --- a/src/soa_builder/web/templates/edit.html +++ b/src/soa_builder/web/templates/edit.html @@ -284,6 +284,63 @@

Matrix: {% if timeline_instances and timeline_instances[0].timeline_name %}{ {% endif %} {% endfor %} + +{% if footnotes %} +
+ {% for fn in footnotes %} +
+ {{ fn.name }}: + {{ fn.text | safe }} +
+ {% endfor %} +
+{% endif %} + +
+ Manage Footnotes +
+ {% if footnotes %} + + + + + + + + + + + {% for fn in footnotes %} + + + + + + + + + + + + {% endfor %} + +
UIDNameLabelDescriptionText (XHTML)
{{ fn.footnote_uid }} +
+ +
+
+ {% endif %} +
+ + + + + +
+
+
Date: Thu, 12 Mar 2026 14:20:12 -0400 Subject: [PATCH 05/27] Added cell superscript functionality to the SOA MAtrix to support footnotes --- src/soa_builder/web/app.py | 163 +++++++++++++++++++++--- src/soa_builder/web/migrate_database.py | 17 ++- src/soa_builder/web/templates/edit.html | 21 +-- 3 files changed, 172 insertions(+), 29 deletions(-) diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index aa008550..ea8c108c 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -72,6 +72,7 @@ _migrate_add_soa_id_indexes, _migrate_add_footnote_table, _migrate_add_footnote_audit_table, + _migrate_matrix_cells_add_superscript, ) from .routers import activities as activities_router from .routers import arms as arms_router @@ -206,6 +207,7 @@ def _configure_logging(): _migrate_add_soa_id_indexes() _migrate_add_footnote_table() _migrate_add_footnote_audit_table() +_migrate_matrix_cells_add_superscript() # Include routers @@ -1117,12 +1119,13 @@ def _fetch_matrix(soa_id: int): ] cur.execute( """ - SELECT instance_id, activity_id, status FROM matrix_cells WHERE soa_id=? AND instance_id IS NOT NULL + SELECT instance_id, activity_id, status, superscript FROM matrix_cells WHERE soa_id=? AND instance_id IS NOT NULL """, (soa_id,), ) cells = [ - dict(instance_id=r[0], activity_id=r[1], status=r[2]) for r in cur.fetchall() + dict(instance_id=r[0], activity_id=r[1], status=r[2], superscript=r[3]) + for r in cur.fetchall() ] conn.close() return instances, activities, cells @@ -3362,6 +3365,32 @@ def set_cell_instance(soa_id: int, payload: dict): return {"cell_id": cid, "status": status} +def _render_cell_td( + soa_id: int, + instance_id: int, + activity_id: int, + status: str, + superscript: str | None, +) -> str: + """Build the HTML for a matrix cell, including superscript and edit button.""" + if status == "X": + sup_html = f"{superscript}" if superscript else "" + edit_btn = ( + f'\u270e' + ) + content = f"X{sup_html}{edit_btn}" + else: + content = "" + return ( + f'{content}' + ) + + @app.post("/ui/soa/{soa_id}/toggle_cell_instance", response_class=HTMLResponse) def ui_toggle_cell_instance( request: Request, @@ -3379,16 +3408,11 @@ def ui_toggle_cell_instance( (soa_id, instance_id, activity_id), ) row = cur.fetchone() - if row and row[0] == "X": - cur.execute("DELETE FROM matrix_cells WHERE id=?", (row[1],)) - conn.commit() - conn.close() - current = "" - elif row: + if row: cur.execute("DELETE FROM matrix_cells WHERE id=?", (row[1],)) conn.commit() conn.close() - current = "" + return HTMLResponse(_render_cell_td(soa_id, instance_id, activity_id, "", None)) else: cur.execute( "INSERT INTO matrix_cells (soa_id, instance_id, activity_id, status) VALUES (?,?,?,?)", @@ -3396,9 +3420,9 @@ def ui_toggle_cell_instance( ) conn.commit() conn.close() - current = "X" - cell_html = f'{current}' - return HTMLResponse(cell_html) + return HTMLResponse( + _render_cell_td(soa_id, instance_id, activity_id, "X", None) + ) # API endpoint for exporting the Matrix as XLSX @@ -4346,6 +4370,9 @@ def ui_edit(request: Request, soa_id: int): activities_page = activities # Build cell lookup cell_map = {(c["instance_id"], c["activity_id"]): c["status"] for c in cells} + superscript_map = { + (c["instance_id"], c["activity_id"]): c.get("superscript") for c in cells + } concepts = fetch_biomedical_concepts() activity_ids = [a["id"] for a in activities_page] activity_concepts = {} @@ -4706,6 +4733,7 @@ def ui_edit(request: Request, soa_id: int): "instances_by_timeline": instances_by_timeline, "default_timeline": default_timeline, "footnotes": footnotes, + "superscript_map": superscript_map, }, ) @@ -5646,7 +5674,9 @@ def ui_toggle_cell( cur.execute("DELETE FROM matrix_cells WHERE id=?", (row[1],)) conn.commit() conn.close() - current = "" + return HTMLResponse( + _render_cell_td(soa_id, int(instance_id), activity_id, "", None) + ) else: cur.execute( "INSERT INTO matrix_cells (soa_id, instance_id, activity_id, status) VALUES (?,?,?,?)", @@ -5654,12 +5684,9 @@ def ui_toggle_cell( ) conn.commit() conn.close() - current = "X" - cell_html = ( - f'{current}' - ) + return HTMLResponse( + _render_cell_td(soa_id, int(instance_id), activity_id, "X", None) + ) else: # Legacy visit-based toggle if visit_id is None: @@ -5683,6 +5710,7 @@ def ui_toggle_cell( conn.commit() conn.close() current = "X" + # Legacy path: visit-based cells don't have superscript support cell_html = ( f' for superscript inline editing.""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT superscript FROM matrix_cells WHERE soa_id=? AND instance_id=? AND activity_id=?", + (soa_id, instance_id, activity_id), + ) + row = cur.fetchone() + conn.close() + if not row: + raise HTTPException(404, "Cell not found") + sup_val = row[0] or "" + html = ( + f'' + f"X" + f'
' + f'' + f'' + f"
" + f'✕' + f"" + ) + return HTMLResponse(html) + + +@app.post( + "/ui/soa/{soa_id}/cell_superscript/{instance_id}/{activity_id}", + response_class=HTMLResponse, +) +def ui_cell_superscript_save( + request: Request, + soa_id: int, + instance_id: int, + activity_id: int, + superscript: Optional[str] = Form(None), +): + """Save superscript value for a cell and return rendered .""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + # Normalise empty string to NULL + sup_val = superscript.strip() if superscript else None + conn = _connect() + cur = conn.cursor() + cur.execute( + "UPDATE matrix_cells SET superscript=? WHERE soa_id=? AND instance_id=? AND activity_id=?", + (sup_val, soa_id, instance_id, activity_id), + ) + conn.commit() + conn.close() + return HTMLResponse(_render_cell_td(soa_id, instance_id, activity_id, "X", sup_val)) + + +@app.get( + "/ui/soa/{soa_id}/cell_superscript_view/{instance_id}/{activity_id}", + response_class=HTMLResponse, +) +def ui_cell_superscript_view( + request: Request, + soa_id: int, + instance_id: int, + activity_id: int, +): + """Return rendered (view-mode) — used for cancel.""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT status, superscript FROM matrix_cells WHERE soa_id=? AND instance_id=? AND activity_id=?", + (soa_id, instance_id, activity_id), + ) + row = cur.fetchone() + conn.close() + status = row[0] if row else "" + sup_val = row[1] if row else None + return HTMLResponse( + _render_cell_td(soa_id, instance_id, activity_id, status or "", sup_val) + ) + + # UI endpoint for associating a Transition Start Rule with Visit/Encounter (visit.transitionStartRule) @app.post( "/ui/soa/{soa_id}/set_visit_transition_start_rule", response_class=HTMLResponse diff --git a/src/soa_builder/web/migrate_database.py b/src/soa_builder/web/migrate_database.py index 6f043798..27fe5983 100644 --- a/src/soa_builder/web/migrate_database.py +++ b/src/soa_builder/web/migrate_database.py @@ -1163,7 +1163,7 @@ def _migrate_add_footnote_table(): id INTEGER PRIMARY KEY AUTOINCREMENT, soa_id INT, footnote_uid TEXT NOT NULL, - name TEXT NOT NULL, + name TEXT NOT NULL, label TEXT, description TEXT, text TEXT, @@ -1199,3 +1199,18 @@ def _migrate_add_footnote_audit_table(): logger.info("_migrate_add_footnote_audit_table created footnote_audit table") except Exception as e: logger.warning("_migrate_add_footnote_audit_table failed: %s", e) + + +def _migrate_matrix_cells_add_superscript(): + """Add superscript TEXT column to matrix_cells if missing.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute("PRAGMA table_info(matrix_cells)") + if "superscript" not in {r[1] for r in cur.fetchall()}: + cur.execute("ALTER TABLE matrix_cells ADD COLUMN superscript TEXT") + conn.commit() + logger.info("Added superscript column to matrix_cells") + conn.close() + except Exception as e: + logger.warning("matrix_cells superscript migration failed: %s", e) diff --git a/src/soa_builder/web/templates/edit.html b/src/soa_builder/web/templates/edit.html index b5ac7e81..29da9ed1 100644 --- a/src/soa_builder/web/templates/edit.html +++ b/src/soa_builder/web/templates/edit.html @@ -258,7 +258,7 @@

Matrix: {% if timeline_instances and timeline_instances[0].timeline_name %}{ {% for a in activities %} - {{ a.name }} + {{ a.label }} {% set concepts_list = activity_concepts.get(a.id, []) %} {% set selected_list = concepts_list %} {% set selected_codes = concepts_list | map(attribute='code') | list %} @@ -267,15 +267,15 @@

Matrix: {% if timeline_instances and timeline_instances[0].timeline_name %}{ {% include 'concepts_cell.html' %} {% for inst in timeline_instances %} {% set raw_status = cell_map.get((inst.id, a.id), '') %} - {% set display = 'X' if raw_status == 'X' else '' %} - - {{ display }} + {% if raw_status == 'X' %}X{% if sup_val %}{{ sup_val }}{% endif %}✎{% endif %} {% endfor %} @@ -355,6 +355,9 @@

Matrix: {% if timeline_instances and timeline_instances[0].timeline_name %}{