Skip to content

Commit e4a9be9

Browse files
neoneyeneoneye
authored
Merge pull request #379 from PlanExeOrg/admin-database-page
feat: admin database page with purge and vacuum
2 parents f059057 + 1ac0915 commit e4a9be9

4 files changed

Lines changed: 342 additions & 127 deletions

File tree

‎frontend_multi_user/src/app.py‎

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2006,6 +2006,83 @@ def _get_database_size_info(self) -> dict[str, Any]:
20062006
info["error"] = str(e)
20072007
return info
20082008

2009+
def _get_purge_activity_info(self) -> dict[str, Any]:
2010+
"""Compute how much space each retention level would free by NULLing run_track_activity_jsonl."""
2011+
from sqlalchemy import text
2012+
info: dict[str, Any] = {"error": None, "total_rows": 0, "rows_with_data": 0, "total_data_mb": 0.0, "options": []}
2013+
try:
2014+
with self.db.engine.connect() as conn:
2015+
row = conn.execute(text(
2016+
"SELECT count(*), "
2017+
"count(run_track_activity_jsonl), "
2018+
"coalesce(sum(octet_length(run_track_activity_jsonl)), 0) "
2019+
"FROM task_item"
2020+
)).fetchone()
2021+
if row:
2022+
info["total_rows"] = row[0]
2023+
info["rows_with_data"] = row[1]
2024+
info["total_data_mb"] = round(row[2] / (1024 * 1024), 2)
2025+
2026+
for keep_n in [10, 25, 50, 100, 250, 500]:
2027+
result = conn.execute(text(
2028+
"SELECT coalesce(sum(octet_length(run_track_activity_jsonl)), 0), count(*) "
2029+
"FROM task_item "
2030+
"WHERE run_track_activity_jsonl IS NOT NULL "
2031+
"AND id NOT IN ("
2032+
" SELECT id FROM task_item "
2033+
" ORDER BY timestamp_created DESC "
2034+
" LIMIT :keep_n"
2035+
")"
2036+
), {"keep_n": keep_n}).fetchone()
2037+
if result:
2038+
info["options"].append({
2039+
"keep_n": keep_n,
2040+
"purgeable_rows": result[1],
2041+
"savings_bytes": result[0],
2042+
"savings_mb": round(result[0] / (1024 * 1024), 2),
2043+
})
2044+
except Exception as e:
2045+
logger.exception("Failed to query purge activity info")
2046+
info["error"] = str(e)
2047+
return info
2048+
2049+
def _purge_activity_data(self, keep_n: int) -> dict[str, Any]:
2050+
"""NULL out run_track_activity_jsonl for all rows except the latest keep_n."""
2051+
from sqlalchemy import text
2052+
result: dict[str, Any] = {"error": None, "purged_rows": 0}
2053+
try:
2054+
with self.db.engine.connect() as conn:
2055+
row = conn.execute(text(
2056+
"UPDATE task_item "
2057+
"SET run_track_activity_jsonl = NULL, run_track_activity_bytes = NULL "
2058+
"WHERE run_track_activity_jsonl IS NOT NULL "
2059+
"AND id NOT IN ("
2060+
" SELECT id FROM task_item "
2061+
" ORDER BY timestamp_created DESC "
2062+
" LIMIT :keep_n"
2063+
")"
2064+
), {"keep_n": keep_n})
2065+
result["purged_rows"] = row.rowcount
2066+
conn.commit()
2067+
except Exception as e:
2068+
logger.exception("Failed to purge activity data")
2069+
result["error"] = str(e)
2070+
return result
2071+
2072+
def _vacuum_task_item(self) -> dict[str, Any]:
2073+
"""Run VACUUM FULL on task_item to reclaim disk space."""
2074+
from sqlalchemy import text
2075+
result: dict[str, Any] = {"error": None}
2076+
try:
2077+
with self.db.engine.connect() as conn:
2078+
conn.execution_options(isolation_level="AUTOCOMMIT").execute(
2079+
text("VACUUM FULL task_item")
2080+
)
2081+
except Exception as e:
2082+
logger.exception("Failed to vacuum task_item")
2083+
result["error"] = str(e)
2084+
return result
2085+
20092086
def _build_reconciliation_report(self, max_tasks: int, tolerance_usd: float) -> tuple[list[dict[str, Any]], dict[str, Any]]:
20102087
tasks = (
20112088
PlanItem.query
@@ -2977,13 +3054,28 @@ def admin_reconciliation():
29773054
refresh_seconds=refresh_seconds,
29783055
)
29793056

2980-
@self.app.route('/admin/db-size')
3057+
@self.app.route('/admin/database', methods=['GET', 'POST'])
29813058
@admin_required
2982-
def admin_db_size():
3059+
def admin_database():
3060+
purge_result = None
3061+
vacuum_result = None
3062+
if request.method == 'POST':
3063+
action = request.form.get('action', '')
3064+
if action == 'purge':
3065+
keep_n = int(request.form.get('keep_n', '50') or '50')
3066+
if keep_n not in (10, 25, 50, 100, 250, 500):
3067+
keep_n = 50
3068+
purge_result = self._purge_activity_data(keep_n)
3069+
elif action == 'vacuum':
3070+
vacuum_result = self._vacuum_task_item()
29833071
size_info = self._get_database_size_info()
3072+
purge_info = self._get_purge_activity_info()
29843073
return self.admin.index_view.render(
2985-
"admin/db_size.html",
3074+
"admin/database.html",
29863075
size_info=size_info,
3076+
purge_info=purge_info,
3077+
purge_result=purge_result,
3078+
vacuum_result=vacuum_result,
29873079
)
29883080

29893081
@self.app.route('/ping/stream')

‎frontend_multi_user/templates/admin/database.html‎

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
{% extends 'admin/master.html' %}
2+
3+
{% block head_css %}
4+
{{ super() }}
5+
<style>
6+
.dbsize-wrap {
7+
max-width: 900px;
8+
margin: 0 auto;
9+
padding: 1.5rem;
10+
color: #222;
11+
}
12+
.dbsize-total {
13+
background: #fff;
14+
border: 1px solid #ddd;
15+
border-radius: 6px;
16+
padding: 1.25rem 1.5rem;
17+
margin-bottom: 1.5rem;
18+
display: flex;
19+
gap: 2rem;
20+
align-items: baseline;
21+
}
22+
.dbsize-total .big-number {
23+
font-size: 2rem;
24+
font-weight: 700;
25+
color: #2c3e50;
26+
}
27+
.dbsize-total .label {
28+
color: #666;
29+
font-size: 0.95rem;
30+
}
31+
table.dbsize-table {
32+
width: 100%;
33+
border-collapse: collapse;
34+
background: #fff;
35+
}
36+
.dbsize-table th,
37+
.dbsize-table td {
38+
border: 1px solid #ddd;
39+
padding: 0.5rem 0.75rem;
40+
font-size: 0.9rem;
41+
}
42+
.dbsize-table th {
43+
background: #f7f7f7;
44+
text-align: left;
45+
}
46+
.dbsize-table td.num {
47+
text-align: right;
48+
font-family: monospace;
49+
}
50+
.bar-cell {
51+
width: 120px;
52+
}
53+
.bar {
54+
height: 14px;
55+
background: #3498db;
56+
border-radius: 3px;
57+
min-width: 1px;
58+
}
59+
.error-banner {
60+
margin: 1rem 0;
61+
padding: 0.75rem 1rem;
62+
border-radius: 6px;
63+
border: 1px solid #e0b4b4;
64+
background: #fff6f6;
65+
color: #8a1f1f;
66+
font-weight: 600;
67+
}
68+
.success-banner {
69+
margin: 1rem 0;
70+
padding: 0.75rem 1rem;
71+
border-radius: 6px;
72+
border: 1px solid #b4e0b4;
73+
background: #f6fff6;
74+
color: #1f8a1f;
75+
font-weight: 600;
76+
}
77+
.purge-section {
78+
background: #fff;
79+
border: 1px solid #ddd;
80+
border-radius: 6px;
81+
padding: 1.25rem 1.5rem;
82+
margin-top: 1.5rem;
83+
}
84+
.purge-section h3 {
85+
margin-top: 0;
86+
}
87+
.purge-options {
88+
margin: 1rem 0;
89+
}
90+
.purge-option {
91+
padding: 0.5rem 0;
92+
display: flex;
93+
align-items: center;
94+
gap: 0.75rem;
95+
}
96+
.purge-option label {
97+
cursor: pointer;
98+
flex: 1;
99+
}
100+
.savings-badge {
101+
font-family: monospace;
102+
font-size: 0.85rem;
103+
color: #666;
104+
}
105+
.savings-badge.has-savings {
106+
color: #b30000;
107+
font-weight: 600;
108+
}
109+
.purge-stats {
110+
font-size: 0.9rem;
111+
color: #666;
112+
margin-bottom: 0.75rem;
113+
}
114+
.btn-purge {
115+
background: #c0392b;
116+
color: #fff;
117+
border: none;
118+
padding: 0.6rem 1.5rem;
119+
border-radius: 4px;
120+
font-size: 0.95rem;
121+
cursor: pointer;
122+
}
123+
.btn-purge:hover {
124+
background: #a93226;
125+
}
126+
</style>
127+
{% endblock %}
128+
129+
{% block body %}
130+
<div class="dbsize-wrap">
131+
<h2>Database Size</h2>
132+
133+
{% if size_info.error %}
134+
<div class="error-banner">
135+
Error querying database size: {{ size_info.error }}
136+
</div>
137+
{% else %}
138+
<div class="dbsize-total">
139+
<div>
140+
<div class="big-number">{{ size_info.total_mb }} MB</div>
141+
<div class="label">Total database size</div>
142+
</div>
143+
<div>
144+
<div class="label">Database: <strong>{{ size_info.database_name }}</strong></div>
145+
</div>
146+
</div>
147+
148+
{% if size_info.tables %}
149+
<h3>Per-table breakdown</h3>
150+
<table class="dbsize-table">
151+
<thead>
152+
<tr>
153+
<th>Table</th>
154+
<th>Total (MB)</th>
155+
<th>Data (MB)</th>
156+
<th>Indexes (MB)</th>
157+
<th class="bar-cell"></th>
158+
</tr>
159+
</thead>
160+
<tbody>
161+
{% for t in size_info.tables %}
162+
<tr>
163+
<td>{{ t.name }}</td>
164+
<td class="num">{{ t.total_mb }}</td>
165+
<td class="num">{{ t.table_mb }}</td>
166+
<td class="num">{{ t.index_mb }}</td>
167+
<td class="bar-cell">
168+
{% if size_info.tables[0].total_bytes > 0 %}
169+
<div class="bar" style="width: {{ (t.total_bytes / size_info.tables[0].total_bytes * 100) | int }}%"></div>
170+
{% endif %}
171+
</td>
172+
</tr>
173+
{% endfor %}
174+
</tbody>
175+
</table>
176+
{% endif %}
177+
178+
{% if purge_result %}
179+
{% if purge_result.error %}
180+
<div class="error-banner">Purge failed: {{ purge_result.error }}</div>
181+
{% else %}
182+
<div class="success-banner">Purged run_track_activity_jsonl from {{ purge_result.purged_rows }} row(s).</div>
183+
{% endif %}
184+
{% endif %}
185+
186+
{% if purge_info and not purge_info.error %}
187+
<div class="purge-section">
188+
<h3>Purge run_track_activity_jsonl</h3>
189+
<div class="purge-stats">
190+
{{ purge_info.rows_with_data }} of {{ purge_info.total_rows }} tasks have activity data
191+
({{ purge_info.total_data_mb }} MB total)
192+
</div>
193+
<form method="POST" onsubmit="return confirm('This will permanently NULL out run_track_activity_jsonl for older rows. Continue?');">
194+
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
195+
<input type="hidden" name="action" value="purge">
196+
<div class="purge-options">
197+
{% for opt in purge_info.options %}
198+
<div class="purge-option">
199+
<input type="radio" name="keep_n" id="keep_{{ opt.keep_n }}" value="{{ opt.keep_n }}"
200+
{% if opt.keep_n == 50 %}checked{% endif %}>
201+
<label for="keep_{{ opt.keep_n }}">
202+
Keep latest <strong>{{ opt.keep_n }}</strong> tasks
203+
</label>
204+
<span class="savings-badge {% if opt.savings_mb > 0 %}has-savings{% endif %}">
205+
{% if opt.purgeable_rows > 0 %}
206+
purge {{ opt.purgeable_rows }} rows, save {{ opt.savings_mb }} MB
207+
{% else %}
208+
nothing to purge
209+
{% endif %}
210+
</span>
211+
</div>
212+
{% endfor %}
213+
</div>
214+
<button type="submit" class="btn-purge">Purge Activity Data</button>
215+
</form>
216+
</div>
217+
{% elif purge_info and purge_info.error %}
218+
<div class="error-banner">Failed to load purge info: {{ purge_info.error }}</div>
219+
{% endif %}
220+
221+
{% if vacuum_result %}
222+
{% if vacuum_result.error %}
223+
<div class="error-banner">Vacuum failed: {{ vacuum_result.error }}</div>
224+
{% else %}
225+
<div class="success-banner">VACUUM FULL task_item completed. Disk space reclaimed.</div>
226+
{% endif %}
227+
{% endif %}
228+
229+
<div class="purge-section">
230+
<h3>Vacuum</h3>
231+
<p class="purge-stats">
232+
Run <code>VACUUM FULL task_item</code> to rewrite the table and reclaim disk space.
233+
This locks the table briefly.
234+
</p>
235+
<form method="POST" onsubmit="return confirm('This will lock the task_item table while it rewrites it. Continue?');">
236+
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
237+
<input type="hidden" name="action" value="vacuum">
238+
<button type="submit" class="btn-purge" style="background: #2980b9;">Vacuum task_item</button>
239+
</form>
240+
</div>
241+
242+
{% endif %}
243+
</div>
244+
{% endblock %}

0 commit comments

Comments
 (0)