From 47fe9a7ac688faca9cf068af0d5081edb20f9114 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 28 May 2026 08:46:50 +0100 Subject: [PATCH 01/21] Make retention disk-pressure quota-aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shutil.disk_usage reports the underlying volume, which on Synology shares, ZFS datasets, NFS shares, and other quota-bound mounts can be far larger than the slice viofosync is actually allowed to consume. The disk-pressure rule then never fired: a 3 TB share inside a 42 TB volume sat silently at 100% of its quota while the threshold saw 86% of the volume. Adds RECORDINGS_QUOTA_GB. When set, used % is measured against bytes-under-recordings ÷ declared quota; a 60-second tree-walk cache amortises the cost across both the retention sweep and any external consumer that wants the same number. Bookkeeping decrement on each delete keeps the inner sweep loop from triggering a fresh walk per file. Leaving the quota at 0 preserves the legacy filesystem path. The percentage and quota rules now trigger independently: each setting works standalone, both-set uses OR semantics so cleanup runs whenever either is breached. --- tests/test_retention.py | 170 ++++++++++++++++++++++++++++++++++++ web/app.py | 1 + web/routers/settings.py | 1 + web/services/retention.py | 144 +++++++++++++++++++++++++----- web/services/sync_worker.py | 1 + web/settings.py | 2 + web/settings_schema.py | 8 +- web/static/app.js | 17 +++- 8 files changed, 318 insertions(+), 26 deletions(-) diff --git a/tests/test_retention.py b/tests/test_retention.py index f338c14..1d1bea7 100644 --- a/tests/test_retention.py +++ b/tests/test_retention.py @@ -225,6 +225,176 @@ def test_sweep_disk_below_threshold_is_noop(env, monkeypatch) -> None: assert _index_count(db) == 1 +def _patch_quota_scanner(monkeypatch, half_gib: int) -> None: + """Helper: each .MP4 on disk counts as half_gib bytes in the + scanner, and _delete_clip_files returns half_gib so the cache + bookkeeping stays consistent with the live scan.""" + import web.services.retention as ret + ret._size_cache.clear() + monkeypatch.setattr( + ret, "_scan_dir_bytes", + lambda p: len(list(Path(p).rglob("*.MP4"))) * half_gib, + ) + orig_del = ret._delete_clip_files + def del_returning(*a, **kw): + orig_del(*a, **kw) + return half_gib + monkeypatch.setattr(ret, "_delete_clip_files", del_returning) + + +def test_sweep_quota_alone_triggers_at_absolute_gib(env, monkeypatch) -> None: + """quota_gb set, disk_pct=0: trip on bytes-under-recordings ≥ quota.""" + rec, db = env + half_gib = (1 << 30) // 2 + _patch_quota_scanner(monkeypatch, half_gib) + # Filesystem looks empty — if code still consults it, nothing would + # be deleted. Sweep must rely on the quota alone. + monkeypatch.setattr( + "web.services.retention.shutil.disk_usage", + lambda p: DiskUsage(total=10**12, used=0, free=10**12), + ) + + _make_clip(rec, db, basename="A.MP4", ts=100) + _make_clip(rec, db, basename="B.MP4", ts=200) + _make_clip(rec, db, basename="C.MP4", ts=300) + # 3 clips × 0.5 GiB = 1.5 GiB used. Quota = 1 GiB. Delete oldest + # until under 1 GiB (i.e. ≤ 2 clips remaining counts at 1 GiB, still + # equal so loop continues; 1 clip = 0.5 GiB, under, stop). + summary = sweep( + db, str(rec), max_days=0, disk_pct=0, + protect_ro=True, quota_gb=1, _now=86400 * 365, + ) + assert summary["deleted_disk"] == 2 + assert _index_count(db) == 1 + with db.conn() as c: + remaining = c.execute( + "SELECT basename FROM clip_index" + ).fetchone()["basename"] + assert remaining == "C.MP4" + + +def test_sweep_quota_zero_only_runs_filesystem_rule(env, monkeypatch) -> None: + """quota_gb=0 must keep the legacy shutil.disk_usage path intact + and not invoke any tree-scan.""" + rec, db = env + _make_clip(rec, db, basename="A.MP4", ts=100) + + scan_calls = {"n": 0} + import web.services.retention as ret + def fake_scan(p): + scan_calls["n"] += 1 + return 0 + monkeypatch.setattr(ret, "_scan_dir_bytes", fake_scan) + + monkeypatch.setattr( + "web.services.retention.shutil.disk_usage", + lambda p: DiskUsage(total=100, used=50, free=50), + ) + summary = sweep( + db, str(rec), max_days=0, disk_pct=80, + protect_ro=True, quota_gb=0, _now=86400 * 365, + ) + assert summary["deleted_disk"] == 0 + assert scan_calls["n"] == 0 # quota path never engaged + + +def test_sweep_both_rules_or_semantics_fs_pct_fires(env, monkeypatch) -> None: + """Both rules set; only the filesystem-% rule is breached. + Sweep must still run.""" + rec, db = env + half_gib = (1 << 30) // 2 + _patch_quota_scanner(monkeypatch, half_gib) + # 1 clip = 0.5 GiB used vs quota=10 GiB → quota NOT breached. + # Filesystem fake says 99% used vs disk_pct=80 → pct IS breached. + monkeypatch.setattr( + "web.services.retention.shutil.disk_usage", + lambda p: DiskUsage(total=100, used=99, free=1), + ) + _make_clip(rec, db, basename="A.MP4", ts=100) + + summary = sweep( + db, str(rec), max_days=0, disk_pct=80, + protect_ro=False, quota_gb=10, _now=86400 * 365, + ) + # Note: pct stays >= 80 forever in this fake — loop should keep + # going until the only clip is gone (then `if not rows: break`). + assert summary["deleted_disk"] == 1 + assert _index_count(db) == 0 + + +def test_sweep_both_rules_or_semantics_quota_fires(env, monkeypatch) -> None: + """Both rules set; only the quota rule is breached. Sweep must + still run, proving the pct rule isn't gating the quota rule.""" + rec, db = env + half_gib = (1 << 30) // 2 + _patch_quota_scanner(monkeypatch, half_gib) + # 3 clips = 1.5 GiB used vs quota=1 GiB → quota IS breached. + # Filesystem says 10% used vs disk_pct=80 → pct NOT breached. + monkeypatch.setattr( + "web.services.retention.shutil.disk_usage", + lambda p: DiskUsage(total=100, used=10, free=90), + ) + _make_clip(rec, db, basename="A.MP4", ts=100) + _make_clip(rec, db, basename="B.MP4", ts=200) + _make_clip(rec, db, basename="C.MP4", ts=300) + + summary = sweep( + db, str(rec), max_days=0, disk_pct=80, + protect_ro=True, quota_gb=1, _now=86400 * 365, + ) + # 1.5 GiB → 1.0 GiB → 0.5 GiB (under quota), stop. + assert summary["deleted_disk"] == 2 + assert _index_count(db) == 1 + + +def test_sweep_both_rules_zero_is_disabled(env, monkeypatch) -> None: + """Neither rule set → disk-pressure phase is skipped entirely.""" + rec, db = env + _make_clip(rec, db, basename="A.MP4", ts=100) + # Both stubs would scream "over threshold" if asked. + import web.services.retention as ret + monkeypatch.setattr(ret, "_scan_dir_bytes", lambda p: 10**20) + monkeypatch.setattr( + "web.services.retention.shutil.disk_usage", + lambda p: DiskUsage(total=100, used=100, free=0), + ) + summary = sweep( + db, str(rec), max_days=0, disk_pct=0, + protect_ro=True, quota_gb=0, _now=86400 * 365, + ) + assert summary["deleted_disk"] == 0 + assert _index_count(db) == 1 + + +def test_scan_dir_bytes_sums_recursively(tmp_path: Path) -> None: + """The size walker must sum everything under the root, recursively.""" + import web.services.retention as ret + (tmp_path / "a.bin").write_bytes(b"x" * 100) + sub = tmp_path / "sub" + sub.mkdir() + (sub / "b.bin").write_bytes(b"y" * 250) + deeper = sub / "deeper" + deeper.mkdir() + (deeper / "c.bin").write_bytes(b"z" * 50) + assert ret._scan_dir_bytes(str(tmp_path)) == 400 + + +def test_cache_subtract_reflects_deletes_without_rescan(tmp_path: Path) -> None: + """The bookkeeping cache must let the inner loop see deletes + immediately without paying for a tree walk per file.""" + import web.services.retention as ret + (tmp_path / "f.bin").write_bytes(b"x" * 1000) + ret._size_cache.clear() + # Prime cache via a fresh scan. + assert ret._cached_used_bytes(str(tmp_path)) == 1000 + # Simulate a delete freeing 400 bytes. + ret._cache_subtract(str(tmp_path), 400) + # Inner check (no refresh) should see the new total. + assert ret._cached_used_bytes(str(tmp_path)) == 600 + # A forced refresh should ignore the cache and rescan. + assert ret._cached_used_bytes(str(tmp_path), refresh=True) == 1000 + + def test_sweep_disk_skips_ro_when_protected(env, monkeypatch) -> None: rec, db = env _make_clip(rec, db, basename="LOCK.MP4", ts=100, event_type="ro") diff --git a/web/app.py b/web/app.py index 51fb465..a1377ec 100644 --- a/web/app.py +++ b/web/app.py @@ -130,6 +130,7 @@ async def _background_retention() -> None: max_days=s.retention_max_days, disk_pct=s.retention_disk_pct, protect_ro=s.retention_protect_ro, + quota_gb=s.recordings_quota_gb, ) except Exception: # pragma: no cover — non-fatal log.exception("startup retention sweep failed") diff --git a/web/routers/settings.py b/web/routers/settings.py index 5c2155b..e06b31b 100644 --- a/web/routers/settings.py +++ b/web/routers/settings.py @@ -40,6 +40,7 @@ def _editable_values(snap) -> dict[str, Any]: "RETENTION_MAX_DAYS": snap.retention_max_days, "RETENTION_DISK_PCT": snap.retention_disk_pct, "RETENTION_PROTECT_RO": snap.retention_protect_ro, + "RECORDINGS_QUOTA_GB": snap.recordings_quota_gb, "TIMEOUT": int(snap.timeout), "DOWNLOAD_ATTEMPTS": snap.download_attempts, "MAX_DOWNLOAD_ATTEMPTS": snap.max_attempts, diff --git a/web/services/retention.py b/web/services/retention.py index c5d2421..369af10 100644 --- a/web/services/retention.py +++ b/web/services/retention.py @@ -16,6 +16,7 @@ import logging import os import shutil +import time as _time_mod from typing import Optional from ..db import Database @@ -94,6 +95,7 @@ def sweep( max_days: int, disk_pct: int, protect_ro: bool, + quota_gb: int = 0, sink=None, _now: Optional[int] = None, ) -> dict: @@ -146,12 +148,14 @@ def sweep( deleted_time, len(rows), bytes_freed / (1 << 20), ) - # Phase 2: disk-pressure. + # Phase 2: disk-pressure. Two independent triggers, either can + # be set on its own; both is fine and uses OR semantics. deleted_disk = 0 - if disk_pct > 0: + if disk_pct > 0 or quota_gb > 0: deleted_disk, freed_2, protected_2 = _disk_pressure_pass( db, recordings, disk_pct=disk_pct, + quota_gb=quota_gb, protect_ro=protect_ro, sink=sink, ) @@ -175,13 +179,90 @@ def sweep( _BATCH_SIZE = 16 +_SIZE_CACHE_TTL = 60.0 +# path -> (computed_at_monotonic, used_bytes). Bookkeeping cache for +# quota mode: deletes subtract from the cached total so the inner +# bail-out check in _disk_pressure_pass doesn't trigger a fresh tree +# walk after every file. +_size_cache: dict[str, tuple[float, int]] = {} -def _used_pct(recordings: str) -> float: +def _scan_dir_bytes(path: str) -> int: + """Sum of file sizes in ``path``, recursing without crossing mount + points. Used by quota mode in place of ``shutil.disk_usage`` when + the OS-level free-space figure doesn't reflect the actual quota + (e.g. Synology shared folder, ZFS dataset, NFS share).""" + try: + root_dev = os.stat(path).st_dev + except OSError: + return 0 + total = 0 + stack = [path] + while stack: + cur = stack.pop() + try: + with os.scandir(cur) as it: + for entry in it: + try: + st = entry.stat(follow_symlinks=False) + except OSError: + continue + if st.st_dev != root_dev: + continue + if entry.is_dir(follow_symlinks=False): + stack.append(entry.path) + elif entry.is_file(follow_symlinks=False): + total += st.st_size + except OSError: + continue + return total + + +def _cached_used_bytes(path: str, *, refresh: bool = False) -> int: + now = _time_mod.monotonic() + cached = _size_cache.get(path) + if not refresh and cached and (now - cached[0]) < _SIZE_CACHE_TTL: + return cached[1] + used = _scan_dir_bytes(path) + _size_cache[path] = (now, used) + return used + + +def _cache_subtract(path: str, freed: int) -> None: + cached = _size_cache.get(path) + if cached is not None and freed > 0: + _size_cache[path] = (cached[0], max(0, cached[1] - freed)) + + +def _pct_exceeded(recordings: str, disk_pct: int) -> bool: + """Filesystem-percent rule. ``disk_pct == 0`` disables it.""" + if disk_pct <= 0: + return False du = shutil.disk_usage(recordings) if du.total <= 0: - return 0.0 - return du.used / du.total * 100.0 + return False + return (du.used / du.total * 100.0) >= disk_pct + + +def _quota_exceeded(recordings: str, quota_gb: int, *, refresh: bool = False) -> bool: + """Absolute-quota rule. ``quota_gb == 0`` disables it. Reads from + the cached size-walk (decremented in-place by each delete) so the + inner sweep loop doesn't pay for a tree walk per file.""" + if quota_gb <= 0: + return False + return _cached_used_bytes(recordings, refresh=refresh) >= quota_gb * (1 << 30) + + +def _over_threshold( + recordings: str, *, disk_pct: int, quota_gb: int, refresh: bool = False +) -> bool: + """True if EITHER rule is currently breached. Independent triggers + — set the percentage to bound the underlying filesystem, set the + quota to bound bytes-under-recordings, or set both.""" + return ( + _pct_exceeded(recordings, disk_pct) + or _quota_exceeded(recordings, quota_gb, refresh=refresh) + ) def _disk_pressure_pass( @@ -189,26 +270,39 @@ def _disk_pressure_pass( recordings: str, *, disk_pct: int, + quota_gb: int, protect_ro: bool, sink, ) -> tuple[int, int, int]: - """Delete oldest clips first until disk usage is under the - threshold or no more eligible candidates remain. - - Disk usage is re-checked at the top of each batch (cheap - syscall) and again after every individual delete inside the - batch — the inner check lets us bail the moment we drop - under the threshold, avoiding overshoot when a single delete - is already enough. - - If we exit still over-threshold AND ``protect_ro`` is on, - counts the surviving RO clips and reports them as - ``protected`` so an operator can see why disk usage didn't - drop. Returns ``(deleted, bytes_freed, protected)``. + """Delete oldest clips first until both pressure rules are + satisfied or no more eligible candidates remain. + + The two rules are independent: ``disk_pct`` measures the + underlying filesystem (cheap syscall via ``shutil.disk_usage``); + ``quota_gb`` measures bytes under ``recordings`` against a + declared cap (needed for Synology shares / ZFS datasets / NFS + where the OS-level free figure doesn't reflect the real + constraint). Either rule on its own works; if both are set we + keep deleting while either is breached. + + Usage is re-checked at the top of each batch (forced fresh in + quota mode so the loop sees ground truth) and again after every + individual delete inside the batch. The inner check lets us bail + the moment all rules are satisfied, avoiding overshoot when a + single delete is already enough. The quota inner check reads the + bookkeeping cache (decremented by each delete) so we don't walk + the tree per file. + + If we exit still over-threshold AND ``protect_ro`` is on, counts + the surviving RO clips and reports them as ``protected`` so an + operator can see why usage didn't drop. Returns ``(deleted, + bytes_freed, protected)``. """ deleted = 0 bytes_freed = 0 - while _used_pct(recordings) >= disk_pct: + while _over_threshold( + recordings, disk_pct=disk_pct, quota_gb=quota_gb, refresh=True, + ): where = "" if protect_ro: where = "WHERE COALESCE(event_type, '') != 'ro'" @@ -224,15 +318,21 @@ def _disk_pressure_pass( if not rows: break for row in rows: - bytes_freed += _delete_clip_files(row, recordings) + freed = _delete_clip_files(row, recordings) + _cache_subtract(recordings, freed) + bytes_freed += freed _delete_index_row(db, row["id"]) deleted += 1 _broadcast(sink, row["basename"], "disk") - if _used_pct(recordings) < disk_pct: + if not _over_threshold( + recordings, disk_pct=disk_pct, quota_gb=quota_gb, + ): return deleted, bytes_freed, 0 protected = 0 - if protect_ro and _used_pct(recordings) >= disk_pct: + if protect_ro and _over_threshold( + recordings, disk_pct=disk_pct, quota_gb=quota_gb, refresh=True, + ): with db.conn() as c: protected = c.execute( "SELECT COUNT(*) AS n FROM clip_index " diff --git a/web/services/sync_worker.py b/web/services/sync_worker.py index 939798f..7788fd5 100644 --- a/web/services/sync_worker.py +++ b/web/services/sync_worker.py @@ -552,6 +552,7 @@ async def _cycle(self) -> bool: max_days=snap.retention_max_days, disk_pct=snap.retention_disk_pct, protect_ro=snap.retention_protect_ro, + quota_gb=snap.recordings_quota_gb, sink=sink, ) except Exception: # pragma: no cover — non-fatal diff --git a/web/settings.py b/web/settings.py index da52a2a..b04017a 100644 --- a/web/settings.py +++ b/web/settings.py @@ -55,6 +55,7 @@ class Snapshot: retention_max_days: int retention_disk_pct: int retention_protect_ro: bool + recordings_quota_gb: int password_hash: str session_secret: str @@ -223,6 +224,7 @@ def _make_snapshot(self, data: dict) -> Snapshot: retention_max_days=m.RETENTION_MAX_DAYS, retention_disk_pct=m.RETENTION_DISK_PCT, retention_protect_ro=m.RETENTION_PROTECT_RO, + recordings_quota_gb=m.RECORDINGS_QUOTA_GB, password_hash=m.WEB_PASSWORD_HASH, session_secret=m.SESSION_SECRET, host=m.WEB_HOST, diff --git a/web/settings_schema.py b/web/settings_schema.py index 4893538..08ed7ce 100644 --- a/web/settings_schema.py +++ b/web/settings_schema.py @@ -43,6 +43,12 @@ class SettingsModel(BaseModel): RETENTION_MAX_DAYS: int = Field(default=0, ge=0, le=3650) RETENTION_DISK_PCT: int = Field(default=0, ge=0, le=99) RETENTION_PROTECT_RO: bool = True + # When > 0, RETENTION_DISK_PCT is measured against this declared + # quota (in GiB) instead of the filesystem reported by os.statvfs. + # Needed when the recordings directory lives inside a quota-bound + # share (Synology shared folder, ZFS dataset quota, etc.) where the + # OS-level "free space" doesn't reflect the actual constraint. + RECORDINGS_QUOTA_GB: int = Field(default=0, ge=0, le=1_048_576) WEB_HOST: str = "0.0.0.0" WEB_PORT: int = Field(default=8080, ge=1, le=65535) @@ -84,7 +90,7 @@ def _validate_email(cls, v: str) -> str: "ENABLE_SCHEDULED_SYNC", "WEB_HOST", "WEB_PORT", "EXPORT_ENCODER", "NOMINATIM_EMAIL", "GEOCODE_ENABLED", "SYNC_RO_ONLY", "RETENTION_MAX_DAYS", "RETENTION_DISK_PCT", - "RETENTION_PROTECT_RO", + "RETENTION_PROTECT_RO", "RECORDINGS_QUOTA_GB", "DISTANCE_UNITS", "PIP_POSITION", } diff --git a/web/static/app.js b/web/static/app.js index 48ac298..4d5e61f 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -2059,9 +2059,15 @@ function renderArchiveSection(pane) { renderField( pane, "RETENTION_DISK_PCT", - "Trigger cleanup at N% disk usage (0 = disabled)", + "Trigger cleanup at N% of filesystem (0 = disabled)", textInput("RETENTION_DISK_PCT", { type: "number", min: 0, max: 99 }), ); + renderField( + pane, + "RECORDINGS_QUOTA_GB", + "Trigger cleanup at this many GiB of recordings (0 = disabled)", + textInput("RECORDINGS_QUOTA_GB", { type: "number", min: 0, max: 1048576 }), + ); renderField( pane, "RETENTION_PROTECT_RO", @@ -2072,8 +2078,13 @@ function renderArchiveSection(pane) { rnote.className = "hint"; rnote.textContent = "Cleanup runs after each sync cycle. Files older than the day cap are " + - "removed; if disk usage is over the threshold, oldest clips are removed " + - "first until under it. Both settings are optional — leave at 0 to disable."; + "always removed first. The two disk-pressure triggers below are " + + "independent — either or both may be set. Filesystem % is the right " + + "choice when recordings live on a dedicated volume. GiB quota is the " + + "right choice when recordings sit inside a Synology share / ZFS " + + "dataset / other quota-bound mount where the filesystem's reported " + + "free space doesn't reflect the actual limit. If both are set, " + + "cleanup runs whenever either is breached."; pane.appendChild(rnote); } From 0be72fb0e3f3c8ed439269715c774facd3c1b794 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 28 May 2026 08:50:03 +0100 Subject: [PATCH 02/21] =?UTF-8?q?Show=20live=20disk=20usage=20in=20Setting?= =?UTF-8?q?s=20=E2=86=92=20Archive=20Retention?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A small card sitting between the section heading and the retention fields now shows current usage as a percentage with absolute bytes, and a coloured bar that turns red once usage crosses the cleanup threshold. A vertical marker on the bar shows where the threshold sits. The card reports the same number the retention sweep evaluates against — quota mode when RECORDINGS_QUOTA_GB is set, filesystem mode otherwise. Both consumers share retention.py's 60-second tree-walk cache, so opening the Settings page costs at most one walk per minute. * GET /api/storage/usage returns {mode, used_bytes, total_bytes, used_pct, threshold_pct, max_days} — session-authed, read-only. * retention.disk_used_pct() is a new public helper that returns Optional[float] (None on error) so external consumers can show "unknown" rather than a misleading 0%. * renderArchiveSection appends the card and kicks an initial refresh. * CSS namespaced under .storage-usage* — no overlap with existing selectors. --- tests/test_storage_endpoint.py | 128 +++++++++++++++++++++++++++++++++ web/app.py | 2 + web/routers/storage.py | 62 ++++++++++++++++ web/services/retention.py | 29 ++++++++ web/static/app.js | 76 ++++++++++++++++++++ web/static/styles.css | 53 ++++++++++++++ 6 files changed, 350 insertions(+) create mode 100644 tests/test_storage_endpoint.py create mode 100644 web/routers/storage.py diff --git a/tests/test_storage_endpoint.py b/tests/test_storage_endpoint.py new file mode 100644 index 0000000..8f8c051 --- /dev/null +++ b/tests/test_storage_endpoint.py @@ -0,0 +1,128 @@ +"""Tests for GET /api/storage/usage.""" +from __future__ import annotations + +import pytest + + +@pytest.fixture +def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + yield c + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def test_usage_endpoint_requires_session(tmp_config_dir, tmp_recordings_dir, + monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + app = create_app() + with TestClient(app) as c: + r = c.get("/api/storage/usage") + assert r.status_code == 401 + + +def test_usage_filesystem_mode_default(logged_in_client): + """No quota set → reports against the filesystem.""" + r = logged_in_client.get("/api/storage/usage") + assert r.status_code == 200 + body = r.json() + assert body["mode"] == "filesystem" + assert body["total_bytes"] > 0 + assert body["used_bytes"] >= 0 + assert 0 <= body["used_pct"] <= 100 + + +def test_usage_quota_mode_reports_against_declared_quota( + logged_in_client, tmp_recordings_dir, +): + """Setting RECORDINGS_QUOTA_GB switches to quota mode.""" + from web import settings as settings_mod + from web.services import retention + + rec = tmp_recordings_dir + (rec / "clip.MP4").write_bytes(b"\0" * (2 << 20)) + retention._size_cache.clear() + + p = settings_mod.get_provider() + p.update({"RECORDINGS_QUOTA_GB": 1}, actor="test") + + r = logged_in_client.get("/api/storage/usage") + assert r.status_code == 200 + body = r.json() + assert body["mode"] == "quota" + assert body["total_bytes"] == 1 << 30 + # 2 MiB used → ~0.2% of a 1 GiB quota + assert 0 < body["used_pct"] < 1 + + +def test_usage_includes_threshold_when_set( + tmp_config_dir, tmp_recordings_dir, monkeypatch, +): + """Pre-seed the threshold in config.json BEFORE app startup so + we don't fire the settings-change subscribers mid-test.""" + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + data["RETENTION_DISK_PCT"] = 80 + data["RETENTION_MAX_DAYS"] = 30 + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + + with TestClient(create_app()) as c: + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + body = c.get("/api/storage/usage").json() + settings_mod.reset_for_tests() + + assert body["threshold_pct"] == 80 + assert body["max_days"] == 30 + + +def test_usage_threshold_null_when_disabled(logged_in_client): + # Defaults are 0 for both; the fixture doesn't change them. + body = logged_in_client.get("/api/storage/usage").json() + assert body["threshold_pct"] is None + assert body["max_days"] is None diff --git a/web/app.py b/web/app.py index a1377ec..d5f7730 100644 --- a/web/app.py +++ b/web/app.py @@ -30,6 +30,7 @@ from .routers import queue as queue_router from .routers import settings as settings_router from .routers import setup as setup_router +from .routers import storage as storage_router from .services import retention as _ret_mod from .services import scanner from .services.exporter import ( @@ -210,6 +211,7 @@ def create_app() -> FastAPI: app.include_router(progress_router.router) app.include_router(settings_router.router) app.include_router(setup_router.router) + app.include_router(storage_router.router) # Static SPA — served at / with an explicit index.html fall-through # so the SPA's hash-router owns everything that isn't /api/*. diff --git a/web/routers/storage.py b/web/routers/storage.py new file mode 100644 index 0000000..d63e56c --- /dev/null +++ b/web/routers/storage.py @@ -0,0 +1,62 @@ +"""Storage status — current usage against quota or filesystem. + +Read-only, session-authenticated, intentionally separate from +``/api/settings`` so a tree-walk doesn't piggy-back on every settings +read. The tree walk itself is cached for 60s by ``retention.py`` so +the cost amortises across the MQTT sensor poll, the UI poll, and the +retention sweep. +""" +from __future__ import annotations + +import shutil + +from fastapi import APIRouter, Depends, Request + +from ..auth import require_session +from ..services import retention as _ret + + +router = APIRouter(prefix="/api/storage", tags=["storage"], + dependencies=[Depends(require_session)]) + + +@router.get("/usage") +def get_usage(request: Request) -> dict: + """Current used-% with the same logic the retention sweep applies. + + Quota mode (``RECORDINGS_QUOTA_GB > 0``) reports the slice viofosync + is allowed to consume; filesystem mode reports the underlying + volume. The UI shows whichever rule actually governs the install. + """ + snap = request.app.state.settings_provider.get() + quota_gb = snap.recordings_quota_gb or 0 + + used_bytes = 0 + total_bytes = 0 + if quota_gb > 0: + used_bytes = _ret._cached_used_bytes(snap.recordings) + total_bytes = quota_gb * (1 << 30) + mode = "quota" + else: + try: + du = shutil.disk_usage(snap.recordings) + used_bytes = du.used + total_bytes = du.total + except (OSError, FileNotFoundError): + used_bytes = 0 + total_bytes = 0 + mode = "filesystem" + + if total_bytes > 0: + used_pct = round(100.0 * used_bytes / total_bytes, 1) + else: + used_pct = None + + return { + "mode": mode, + "used_bytes": used_bytes, + "total_bytes": total_bytes, + "used_pct": used_pct, + "threshold_pct": snap.retention_disk_pct or None, + "max_days": snap.retention_max_days or None, + } diff --git a/web/services/retention.py b/web/services/retention.py index 369af10..6c626a2 100644 --- a/web/services/retention.py +++ b/web/services/retention.py @@ -234,6 +234,35 @@ def _cache_subtract(path: str, freed: int) -> None: _size_cache[path] = (cached[0], max(0, cached[1] - freed)) +def disk_used_pct(recordings: str, quota_gb: int = 0) -> Optional[float]: + """Public helper for consumers (MQTT, status APIs) that want the + same used-% the retention sweep evaluates against. + + Returns ``None`` when the recordings path is missing or the + quota is meaningless — callers should treat that as Unknown + rather than 0%. Reuses the 60-second tree-walk cache so two + consumers a minute apart don't double-walk. + + For display only — the sweep itself uses ``_pct_exceeded`` and + ``_quota_exceeded`` to decide whether each rule is currently + breached, so a quota set without a percentage threshold (or + vice versa) triggers independently. + """ + if quota_gb > 0: + used = _cached_used_bytes(recordings) + limit = quota_gb * (1 << 30) + if limit <= 0: + return None + return used / limit * 100.0 + try: + du = shutil.disk_usage(recordings) + except (OSError, FileNotFoundError): + return None + if du.total <= 0: + return None + return du.used / du.total * 100.0 + + def _pct_exceeded(recordings: str, disk_pct: int) -> bool: """Filesystem-percent rule. ``disk_pct == 0`` disables it.""" if disk_pct <= 0: diff --git a/web/static/app.js b/web/static/app.js index 4d5e61f..cfa327e 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -2050,6 +2050,24 @@ function renderArchiveSection(pane) { h.style.marginTop = "24px"; pane.appendChild(h); + // Live usage card. Sits between the heading and the threshold input + // so users can see what their threshold is being measured against. + const usageCard = document.createElement("div"); + usageCard.className = "storage-usage"; + usageCard.innerHTML = ` +
+ Current usage + +
+
+
+ +
+

+ `; + pane.appendChild(usageCard); + refreshStorageUsage(usageCard); + renderField( pane, "RETENTION_MAX_DAYS", @@ -2088,6 +2106,64 @@ function renderArchiveSection(pane) { pane.appendChild(rnote); } +// ---- Storage usage card ---- + +function _fmtBytes(n) { + if (!n || n <= 0) return "0 B"; + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let i = 0; + while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; } + return (n < 10 ? n.toFixed(1) : Math.round(n)) + " " + units[i]; +} + +async function refreshStorageUsage(card) { + if (!card || !card.isConnected) return; + let body; + try { + body = await api("/api/storage/usage"); + } catch (_) { + return; + } + if (!card.isConnected) return; + + const value = card.querySelector(".storage-usage-value"); + const fill = card.querySelector(".storage-usage-fill"); + const mark = card.querySelector(".storage-usage-threshold"); + const mode = card.querySelector(".storage-usage-mode"); + + if (body.total_bytes <= 0 || body.used_pct === null) { + value.textContent = "Unavailable"; + fill.style.width = "0%"; + mark.style.display = "none"; + mode.textContent = body.mode === "quota" + ? "Quota set but path unreadable." + : "Could not read filesystem stats."; + return; + } + + const pct = body.used_pct; + value.textContent = + `${pct.toFixed(1)}% — ${_fmtBytes(body.used_bytes)} of ${_fmtBytes(body.total_bytes)}`; + fill.style.width = Math.min(100, pct) + "%"; + + // Tint the fill if we're at or past the cleanup threshold. + fill.classList.toggle("over-threshold", + body.threshold_pct != null && pct >= body.threshold_pct); + + if (body.threshold_pct != null && body.threshold_pct > 0) { + mark.style.display = "block"; + mark.style.left = Math.min(100, body.threshold_pct) + "%"; + mark.title = `Cleanup threshold: ${body.threshold_pct}%`; + } else { + mark.style.display = "none"; + } + + mode.textContent = body.mode === "quota" + ? `Measured against your ${_fmtBytes(body.total_bytes)} quota.` + : `Measured against the filesystem containing recordings.`; +} + + function renderWebSection(pane) { renderField(pane, "WEB_HOST", "Bind host", textInput("WEB_HOST")); renderField(pane, "WEB_PORT", "Listen port", diff --git a/web/static/styles.css b/web/static/styles.css index f9947bf..4151e93 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -1470,3 +1470,56 @@ main { transition-duration: 0.01ms !important; } } + +/* ============================================================ + * Storage usage card (Settings → Archive Retention) + * ============================================================ */ +.storage-usage { + margin: 0.5rem 0 1.25rem; + padding: 0.75rem 0.9rem; + background: var(--bg-elev, rgba(0,0,0,0.04)); + border: 1px solid var(--border, rgba(0,0,0,0.1)); + border-radius: 6px; +} +.storage-usage-row { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 0.5rem; + margin-bottom: 0.5rem; +} +.storage-usage-label { + font-weight: 600; + font-size: 0.9rem; +} +.storage-usage-value { + font-variant-numeric: tabular-nums; + font-size: 0.9rem; + color: var(--muted, #666); +} +.storage-usage-bar { + position: relative; + height: 8px; + background: var(--bar-track, rgba(0,0,0,0.08)); + border-radius: 4px; + overflow: hidden; +} +.storage-usage-fill { + height: 100%; + background: var(--accent, #19a974); + transition: width 0.4s ease, background 0.3s ease; +} +.storage-usage-fill.over-threshold { + background: #d33; +} +.storage-usage-threshold { + position: absolute; + top: -2px; + bottom: -2px; + width: 2px; + background: rgba(0,0,0,0.55); + pointer-events: auto; +} +.storage-usage-mode { + margin: 0.5rem 0 0; +} From bf10b39960a60a566c177c65d51a77e123e43de0 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 28 May 2026 08:53:26 +0100 Subject: [PATCH 03/21] Add MQTT publishing with Home Assistant auto-discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publish viofosync state and accept actions over MQTT, with full HA discovery so sensors and buttons appear automatically with zero HA-side YAML. Surface: - One device card "Viofosync" with 11 entities - binary_sensor.viofosync_dashcam (connectivity) - sensor.viofosync_sync_status (stopped/idle/paused/downloading) - sensor.viofosync_queue_pending - sensor.viofosync_last_downloaded_clip (timestamp) - sensor.viofosync_disk_used (% of the tighter of quota or filesystem — same number the Settings card shows) - disabled-by-default: queue_failed, queue_downloading, current_filename, current_progress, total_clips - 6 action buttons: start_sync, pause_sync, skip_current, refresh_queue, retry_failed, rescan_archive - 1 parameterised command topic: /cmd/prioritize_recent {"hours": N} Architecture: - Single MqttService managed by the FastAPI lifespan; aiomqtt v2 client in one asyncio task with exponential backoff reconnect - Pure-data EntityDef catalog in mqtt_topology.py is the single source of truth for discovery payloads, state extractors, and command handlers - PublishCoalescer suppresses unchanged payloads and rate-limits per topic so idle traffic is zero - LWT + retained availability "online"/"offline" so HA marks all entities Unavailable within ~45s of an unclean disconnect - Hot-reload of broker settings; node-rename publishes empty-payload discovery deletes so HA forgets the old topology cleanly - Settings page gains an MQTT panel with broker host/port/auth, TLS, topic prefix, discovery prefix, QoS, a Test connection probe, and a live status dot Tests: 60+ unit tests plus an end-to-end test against an in-process amqtt broker covering connect, discovery republish, state publication, and command dispatch. Docs: README section + CHANGELOG entry. --- CHANGELOG.md | 10 + README.md | 59 ++- pyproject.toml | 16 +- requirements-dev.txt | 1 + requirements.txt | 1 + tests/conftest.py | 59 +++ tests/test_mqtt_command_routing.py | 146 ++++++++ tests/test_mqtt_discovery_payload.py | 172 +++++++++ tests/test_mqtt_e2e.py | 149 ++++++++ tests/test_mqtt_events.py | 153 ++++++++ tests/test_mqtt_hub_bridge.py | 21 ++ tests/test_mqtt_publication.py | 119 ++++++ tests/test_mqtt_settings.py | 97 +++++ tests/test_mqtt_settings_reload.py | 85 +++++ tests/test_mqtt_state_extraction.py | 314 ++++++++++++++++ tests/test_mqtt_status.py | 77 ++++ tests/test_mqtt_test_endpoint.py | 73 ++++ tests/test_mqtt_topology.py | 95 +++++ tests/test_queue_retry_failed.py | 41 +++ tests/test_storage_endpoint.py | 23 ++ web/app.py | 30 +- web/routers/archive.py | 1 + web/routers/mqtt.py | 65 ++++ web/routers/queue.py | 3 + web/routers/settings.py | 11 + web/services/mqtt.py | 523 +++++++++++++++++++++++++++ web/services/mqtt_state.py | 154 ++++++++ web/services/mqtt_topology.py | 384 ++++++++++++++++++++ web/services/queue.py | 37 ++ web/services/scanner.py | 15 +- web/services/sync_worker.py | 4 + web/settings.py | 23 ++ web/settings_schema.py | 55 ++- web/static/app.js | 103 ++++++ web/static/index.html | 1 + web/static/styles.css | 23 +- 36 files changed, 3132 insertions(+), 11 deletions(-) create mode 100644 tests/test_mqtt_command_routing.py create mode 100644 tests/test_mqtt_discovery_payload.py create mode 100644 tests/test_mqtt_e2e.py create mode 100644 tests/test_mqtt_events.py create mode 100644 tests/test_mqtt_hub_bridge.py create mode 100644 tests/test_mqtt_publication.py create mode 100644 tests/test_mqtt_settings.py create mode 100644 tests/test_mqtt_settings_reload.py create mode 100644 tests/test_mqtt_state_extraction.py create mode 100644 tests/test_mqtt_status.py create mode 100644 tests/test_mqtt_test_endpoint.py create mode 100644 tests/test_mqtt_topology.py create mode 100644 tests/test_queue_retry_failed.py create mode 100644 web/routers/mqtt.py create mode 100644 web/services/mqtt.py create mode 100644 web/services/mqtt_state.py create mode 100644 web/services/mqtt_topology.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4469721..b2d85f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +### Added +- MQTT publishing with Home Assistant auto-discovery. New Settings + panel exposes broker host/port/credentials, TLS, topic prefix, and + discovery prefix. Publishes 12 sensor/binary_sensor entities and 6 + action buttons; idle traffic is zero thanks to per-topic change + detection and coalescing. LWT keeps HA's view consistent with + viofosync's actual state. + ## v2.1 — 2026-05-16 ### Fixed diff --git a/README.md b/README.md index 683a681..c16a43d 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,13 @@ Self-hosted web app for syncing, browsing, and exporting recordings from a Viofo ## Features -- **Archive browser** — view clips grouped by day, front/rear pairs, on-demand thumbnails, in-browser playback, kind filters (Driving / Parking / Read-only), GPS-maps toggle for low-bandwidth browsing. -- **GPS journeys** — Leaflet + OSM map per trip, automatic stop detection splits a day into journeys, reverse-geocoded start/end labels (e.g. *Whitegate → Sandiway*). -- **Exports** — select clip pairs, render joined front-only, rear-only, or picture-in-picture videos with ffmpeg. Hardware H.264 (videotoolbox / nvenc / qsv / vaapi) when available, software libx264 fallback. -- **Download manager** — live progress, reorderable queue, reachability badge, transient timeouts re-queue instead of burning retries. -- **Auto-delete from dashcam** *(optional)* — clears each clip from the device once it's downloaded and verified. -- **Settings page** — runtime settings hot-reload rather than Docker env vars; only `WEB_HOST`/`WEB_PORT` need a restart. +- **Archive browser** — clips grouped by day, paired front/rear, in-browser playback. +- **GPS journeys** — map per trip with auto-split stops and reverse-geocoded place names. +- **Exports** — joined front, rear, or picture-in-picture videos via ffmpeg. +- **Download manager** — live progress and a reorderable queue. +- **Auto-delete from dashcam** *(optional)* — frees SD card space once a clip is safely downloaded. +- **Settings page** — runtime settings hot-reload; no Docker env vars to fiddle with. +- **Home Assistant** *(optional)* — auto-discovered sensors and buttons via MQTT. ## Hardware @@ -57,6 +58,52 @@ The only Docker-level env vars are: App-level settings (sync interval, dashcam IP, encoder, geocoding email, web port, retention, password, auto-delete, etc.) are editable on the **Settings** page. Advanced users can hand-edit `/config/config.json` between restarts; the schema lives in `[web/settings_schema.py](web/settings_schema.py)`. +## Home Assistant via MQTT + +viofosync can publish state and accept actions over MQTT, with full +Home Assistant auto-discovery — no HA-side YAML required. + +Enable on the Settings page → MQTT. You'll need: + +- A reachable MQTT broker (Mosquitto, HA's built-in broker, EMQX, etc.). +- Broker host + port. Optional username, password, and TLS. +- A `Node ID` (default `viofosync`) — used as the topic prefix and as + the `node_id` slot in HA discovery topics. Letters, digits, and `_` + only. Set a distinct value per instance if you run more than one. + +When MQTT is on, viofosync publishes: + +- **Discovery configs** under `homeassistant/{component}/{node_id}/{object_id}/config` + (retained) so HA picks them up immediately. +- **State** under `{node_id}/{object_id}/state` (retained, event-driven, + no idle traffic). +- **Availability** to `{node_id}/availability` (`online` / `offline`), + with LWT so HA marks every entity Unavailable within ~45s of an + unclean disconnect. + +### Sensors and buttons + +Enabled by default in HA: dashcam connectivity, sync status +(`stopped` / `idle` / `paused` / `downloading`), queue pending, last +downloaded clip, disk used, and six action buttons +(start/pause/skip/refresh/retry-failed/rescan). + +Disabled-by-default (still created — enable per-entity in HA): queue +failed, queue downloading, current filename, current progress, total +clips. + +### Parameterised command + +For "prioritize the last N hours", publish to +`{node_id}/cmd/prioritize_recent` with payload `{"hours": 0.5}` (HA's +`mqtt.publish` service works). `hours` must be in (0, 168]. + +### Security notes + +- The MQTT password is stored in `config.json` in plaintext, alongside + the bcrypt hash of the admin password and the session secret. The + same access controls already apply to that file. + ## Reverse geocoding Journey and stop cards display their start/end as *"Street, Town"* via Nominatim (OpenStreetMap). Lookups are rate-limited to 1/second per [Nominatim's usage policy](https://operations.osmfoundation.org/policies/nominatim/) and cached in the `geocode_cache` table (coords rounded to 3 d.p., ≈110 m). Set **Nominatim email** in Settings → GPS & Geocoding to identify your install per OSM's terms; toggle the **GPS maps** filter off on the Archive page to skip the Leaflet + Nominatim machinery entirely for low-bandwidth browsing. diff --git a/pyproject.toml b/pyproject.toml index c0cb7de..04ae87f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,21 @@ [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" -filterwarnings = ["error"] +filterwarnings = [ + "error", + # amqtt broker thread leaves AF_UNIX sockets and its event loop + # unclosed on GC during session teardown — these are benign. + "ignore::ResourceWarning:amqtt", + "ignore:unclosed event loop:ResourceWarning", + "ignore:unclosed =8.0 pytest-asyncio>=0.23 httpx>=0.27 ruff>=0.4 +amqtt>=0.11.0 diff --git a/requirements.txt b/requirements.txt index 2da26f9..43a733f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ itsdangerous>=2.2 python-multipart>=0.0.9 pydantic>=2.6 httpx>=0.27 +aiomqtt>=2.0,<3.0 diff --git a/tests/conftest.py b/tests/conftest.py index 83ba3f3..6f09e58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,3 +32,62 @@ def tmp_recordings_dir(monkeypatch) -> Iterator[Path]: yield d finally: shutil.rmtree(d, ignore_errors=True) + + +@pytest.fixture(scope="session") +def mqtt_broker(): + """Start an in-process amqtt broker on a random port for the session.""" + import asyncio + import socket + import threading + import warnings + + # Pick a free port + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + + cfg = { + "listeners": { + "default": { + "type": "tcp", + "bind": f"127.0.0.1:{port}", + "max_connections": 50, + }, + }, + "sys_interval": 0, + "auth": {"allow-anonymous": True}, + "topic-check": {"enabled": False}, + } + + loop = asyncio.new_event_loop() + ready = threading.Event() + broker_holder: list = [] + + def _runner(): + asyncio.set_event_loop(loop) + # Suppress amqtt deprecation warnings about old config keys. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from amqtt.broker import Broker + broker = Broker(cfg, loop=loop) + broker_holder.append(broker) + loop.run_until_complete(broker.start()) + ready.set() + loop.run_forever() + loop.close() + + t = threading.Thread(target=_runner, daemon=True, name="amqtt-broker") + t.start() + ready.wait(timeout=5.0) + try: + yield ("127.0.0.1", port) + finally: + broker = broker_holder[0] + + async def _shutdown(): + await broker.shutdown() + + asyncio.run_coroutine_threadsafe(_shutdown(), loop).result(timeout=5.0) + loop.call_soon_threadsafe(loop.stop) + t.join(timeout=5.0) diff --git a/tests/test_mqtt_command_routing.py b/tests/test_mqtt_command_routing.py new file mode 100644 index 0000000..85757a6 --- /dev/null +++ b/tests/test_mqtt_command_routing.py @@ -0,0 +1,146 @@ +"""Command handler factory + routing tests.""" +from __future__ import annotations + +import asyncio +import types + +import pytest + + +def _fake_app(tmp_path): + from web.db import Database + sw_calls = [] + + class FakeSync: + def resume(self): sw_calls.append("resume") + def start(self): sw_calls.append("start") + def kick(self): sw_calls.append("kick") + def pause(self): sw_calls.append("pause") + def skip_current(self): sw_calls.append("skip") + fake_state = types.SimpleNamespace( + sync_worker=FakeSync(), + db=Database(str(tmp_path / "v.db")), + hub=None, # emit_queue_changed no-ops when hub is None + settings_provider=types.SimpleNamespace( + get=lambda: types.SimpleNamespace( + recordings=str(tmp_path), grouping="daily", + ), + ), + ) + fake_state.sync_worker_calls = sw_calls # so the test can read them + return types.SimpleNamespace(state=fake_state, version="0.2.0") + + +@pytest.mark.asyncio +async def test_start_sync_handler(tmp_path): + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + handlers = build_command_handlers(app) + await handlers["start_sync"](b"PRESS") + assert app.state.sync_worker_calls == ["resume", "start", "kick"] + + +@pytest.mark.asyncio +async def test_pause_sync_handler(tmp_path): + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + await build_command_handlers(app)["pause_sync"](b"PRESS") + assert app.state.sync_worker_calls == ["pause"] + + +@pytest.mark.asyncio +async def test_skip_current_handler(tmp_path): + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + await build_command_handlers(app)["skip_current"](b"PRESS") + assert app.state.sync_worker_calls == ["skip"] + + +@pytest.mark.asyncio +async def test_refresh_queue_handler(tmp_path): + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + await build_command_handlers(app)["refresh_queue"](b"PRESS") + assert app.state.sync_worker_calls == ["kick"] + + +@pytest.mark.asyncio +async def test_retry_failed_handler(tmp_path): + import time as _t + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + with app.state.db.write() as c: + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at, attempts) " + "VALUES (?, ?, ?, ?, ?)", + ("a.MP4", "/DCIM", "failed", int(_t.time()), 2), + ) + await build_command_handlers(app)["retry_failed"](b"PRESS") + with app.state.db.conn() as c: + state = c.execute( + "SELECT state FROM download_queue" + ).fetchone()["state"] + assert state == "pending" + assert app.state.sync_worker_calls == ["kick"] + + +@pytest.mark.asyncio +async def test_rescan_archive_handler(tmp_path, monkeypatch): + from web.services import scanner + from web.services.mqtt_topology import build_command_handlers + calls = [] + monkeypatch.setattr(scanner, "scan", + lambda db, dest, grouping, *a, **kw: calls.append((dest, grouping)) or 0) + app = _fake_app(tmp_path) + await build_command_handlers(app)["rescan_archive"](b"PRESS") + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_prioritize_recent_valid_payload(tmp_path): + import time as _t + import json + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + now = int(_t.time()) + with app.state.db.write() as c: + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at, recorded_at) " + "VALUES (?, ?, ?, ?, ?)", + ("recent.MP4", "/DCIM", "pending", now, now - 60), + ) + await build_command_handlers(app)["prioritize_recent"]( + json.dumps({"hours": 0.5}).encode() + ) + with app.state.db.conn() as c: + prio = c.execute( + "SELECT priority FROM download_queue WHERE filename='recent.MP4'" + ).fetchone()["priority"] + assert prio > 0 + assert app.state.sync_worker_calls == ["kick"] + + +@pytest.mark.asyncio +async def test_prioritize_recent_rejects_bad_json(tmp_path, caplog): + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + # Should not raise. + await build_command_handlers(app)["prioritize_recent"](b"not json") + # No worker call, no priority change. + assert app.state.sync_worker_calls == [] + + +@pytest.mark.asyncio +async def test_prioritize_recent_rejects_out_of_range(tmp_path): + import json + from web.services.mqtt_topology import build_command_handlers + app = _fake_app(tmp_path) + await build_command_handlers(app)["prioritize_recent"]( + json.dumps({"hours": 0}).encode() + ) + await build_command_handlers(app)["prioritize_recent"]( + json.dumps({"hours": 200}).encode() + ) + assert app.state.sync_worker_calls == [] diff --git a/tests/test_mqtt_discovery_payload.py b/tests/test_mqtt_discovery_payload.py new file mode 100644 index 0000000..90423bf --- /dev/null +++ b/tests/test_mqtt_discovery_payload.py @@ -0,0 +1,172 @@ +"""Discovery payload builder tests.""" +from __future__ import annotations + + +def _cfg(**overrides): + base = { + "discovery_prefix": "homeassistant", + "node_id": "viofosync", + "version": "0.2.0", + "configuration_url": "http://host:8080/", + } + base.update(overrides) + return base + + +def test_state_topic(): + from web.services.mqtt_topology import build_state_topic + assert build_state_topic("queue_pending", _cfg()) == ( + "viofosync/queue_pending/state" + ) + + +def test_command_topic(): + from web.services.mqtt_topology import build_command_topic + assert build_command_topic("pause_sync", _cfg()) == ( + "viofosync/pause_sync/cmd" + ) + + +def test_discovery_topic_sensor(): + from web.services.mqtt_topology import build_discovery_topic + assert build_discovery_topic( + "sensor", "queue_pending", _cfg(), + ) == "homeassistant/sensor/viofosync/queue_pending/config" + + +def test_discovery_topic_button(): + from web.services.mqtt_topology import build_discovery_topic + assert build_discovery_topic( + "button", "pause_sync", _cfg(), + ) == "homeassistant/button/viofosync/pause_sync/config" + + +def test_availability_topic(): + from web.services.mqtt_topology import build_availability_topic + assert build_availability_topic(_cfg()) == "viofosync/availability" + + +def test_unique_id_includes_node_id(): + from web.services.mqtt_topology import build_unique_id + assert build_unique_id("queue_pending", _cfg()) == ( + "viofosync_viofosync_queue_pending" + ) + assert build_unique_id("queue_pending", _cfg(node_id="garage")) == ( + "viofosync_garage_queue_pending" + ) + + +def test_discovery_payload_for_sensor(): + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, + ) + entity = EntityDef( + object_id="queue_pending", + component="sensor", + name="Queue pending", + icon="mdi:download", + device_class=None, + state_class="measurement", + unit_of_measurement=None, + enabled_by_default=True, + min_publish_interval_s=1.0, + state_fn=None, + command_handler=None, + affected_by_hub_events=(), + ) + payload = build_discovery_payload(entity, _cfg()) + assert payload["name"] == "Queue pending" + assert payload["unique_id"] == "viofosync_viofosync_queue_pending" + assert payload["state_topic"] == "viofosync/queue_pending/state" + assert payload["availability_topic"] == "viofosync/availability" + assert payload["payload_available"] == "online" + assert payload["payload_not_available"] == "offline" + assert payload["state_class"] == "measurement" + assert payload["icon"] == "mdi:download" + assert payload["enabled_by_default"] is True + assert "command_topic" not in payload + # device manifest is present and references the node_id + assert payload["device"]["identifiers"] == ["viofosync_viofosync"] + assert payload["device"]["sw_version"] == "0.2.0" + + +def test_discovery_payload_for_button(): + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, + ) + async def _h(_p): ... + entity = EntityDef( + object_id="pause_sync", + component="button", + name="Pause sync", + icon="mdi:pause", + device_class=None, + state_class=None, + unit_of_measurement=None, + enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, + command_handler=_h, + affected_by_hub_events=(), + ) + payload = build_discovery_payload(entity, _cfg()) + assert payload["command_topic"] == "viofosync/pause_sync/cmd" + # Buttons don't have a state_topic in HA discovery + assert "state_topic" not in payload + + +def test_device_manifest_omits_empty_configuration_url(): + """HA's MQTT discovery rejects the entire message with + 'invalid url for dictionary value' when configuration_url is + present but empty. The builder must omit the key in that case.""" + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, + ) + entity = EntityDef( + object_id="queue_pending", component="sensor", + name="Queue pending", + icon=None, device_class=None, state_class="measurement", + unit_of_measurement=None, + enabled_by_default=True, min_publish_interval_s=1.0, + state_fn=None, command_handler=None, + affected_by_hub_events=(), + ) + payload = build_discovery_payload(entity, _cfg(configuration_url="")) + assert "configuration_url" not in payload["device"] + # Sanity: when present and non-empty, it IS included + payload = build_discovery_payload( + entity, _cfg(configuration_url="http://host:8080/"), + ) + assert payload["device"]["configuration_url"] == "http://host:8080/" + + +def test_discovery_payload_unit_when_set(): + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, + ) + entity = EntityDef( + object_id="disk_used", component="sensor", name="Disk used", + icon=None, device_class=None, state_class="measurement", + unit_of_measurement="%", + enabled_by_default=True, min_publish_interval_s=0.0, + state_fn=None, command_handler=None, + affected_by_hub_events=(), + ) + payload = build_discovery_payload(entity, _cfg()) + assert payload["unit_of_measurement"] == "%" + + +def test_discovery_payload_disabled_by_default(): + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, + ) + entity = EntityDef( + object_id="queue_failed", component="sensor", name="Queue failed", + icon=None, device_class=None, state_class="measurement", + unit_of_measurement=None, + enabled_by_default=False, min_publish_interval_s=1.0, + state_fn=None, command_handler=None, + affected_by_hub_events=(), + ) + payload = build_discovery_payload(entity, _cfg()) + assert payload["enabled_by_default"] is False diff --git a/tests/test_mqtt_e2e.py b/tests/test_mqtt_e2e.py new file mode 100644 index 0000000..3798023 --- /dev/null +++ b/tests/test_mqtt_e2e.py @@ -0,0 +1,149 @@ +"""End-to-end test against an in-process amqtt broker.""" +from __future__ import annotations + +import asyncio +import json +import threading + + +import pytest + + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.filterwarnings("ignore::ResourceWarning") +def test_full_walkthrough(mqtt_broker, tmp_path, monkeypatch): + """Drive a real amqtt broker through discovery, state publish, and commands.""" + import aiomqtt + import bcrypt + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + from fastapi.testclient import TestClient + + host, port = mqtt_broker + + # ------------------------------------------------------------------ + # Configure settings with MQTT enabled. + # ------------------------------------------------------------------ + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + monkeypatch.setenv("CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("RECORDINGS", str(tmp_path / "rec")) + (tmp_path / "rec").mkdir() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + data["MQTT_ENABLED"] = True + data["MQTT_HOST"] = host + data["MQTT_PORT"] = port + data["MQTT_DISCOVERY_ENABLED"] = True + data["MQTT_NODE_ID"] = "viofosync" + data["MQTT_CLIENT_ID"] = "test-e2e" + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + app = create_app() + + # ------------------------------------------------------------------ + # Run an async subscriber in a background thread. + # It collects every MQTT message until told to stop. + # ------------------------------------------------------------------ + received: list[tuple[str, bytes]] = [] + sub_ready = threading.Event() + sub_loop: list[asyncio.AbstractEventLoop] = [] + sub_task_holder: list[asyncio.Task] = [] + + def run_subscriber(): + async def _subscriber(): + async with aiomqtt.Client(hostname=host, port=port) as c: + await c.subscribe("#", qos=1) + sub_ready.set() + async for m in c.messages: + received.append((m.topic.value, bytes(m.payload))) + + async def _run(): + task = asyncio.ensure_future(_subscriber()) + sub_task_holder.append(task) + try: + await task + except (asyncio.CancelledError, Exception): + pass + # Drain any remaining callbacks so sockets are closed cleanly. + await asyncio.sleep(0) + + loop = asyncio.new_event_loop() + sub_loop.append(loop) + try: + loop.run_until_complete(_run()) + except (asyncio.CancelledError, Exception): + pass + finally: + # Cancel all remaining tasks before closing. + pending = asyncio.all_tasks(loop) + for t in pending: + t.cancel() + if pending: + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + loop.close() + + sub_thread = threading.Thread(target=run_subscriber, daemon=True, name="e2e-subscriber") + sub_thread.start() + # Wait for subscriber to connect and subscribe. + sub_ready.wait(timeout=5.0) + + # ------------------------------------------------------------------ + # Run the FastAPI app lifespan via TestClient (sync). + # ------------------------------------------------------------------ + with TestClient(app) as _client: + import time + time.sleep(2.5) # Allow MqttService to connect and publish. + + topics = {t for (t, _p) in received} + assert "viofosync/availability" in topics, ( + f"Expected 'viofosync/availability' in {sorted(topics)}" + ) + assert any("homeassistant/sensor/viofosync/" in t for t in topics), ( + f"No homeassistant discovery topic in {sorted(topics)}" + ) + assert "viofosync/queue_pending/state" in topics, ( + f"Expected 'viofosync/queue_pending/state' in {sorted(topics)}" + ) + + # Drive a command: pause sync. + def publish_sync(topic, payload): + async def _pub(): + async with aiomqtt.Client(hostname=host, port=port) as cmd: + await cmd.publish(topic, payload, qos=1) + asyncio.run(_pub()) + + # Install a spy on the real sync_worker so we can assert pause() fires. + pause_calls: list = [] + original_pause = _client.app.state.sync_worker.pause + _client.app.state.sync_worker.pause = ( + lambda *a, **kw: (pause_calls.append(1), original_pause(*a, **kw))[1] + ) + + publish_sync("viofosync/pause_sync/cmd", b"PRESS") + time.sleep(0.5) + assert len(pause_calls) >= 1, ( + "pause_sync command didn't reach sync_worker.pause" + ) + + # Drive a parameterised command — prioritize_recent. + publish_sync( + "viofosync/cmd/prioritize_recent", + json.dumps({"hours": 1}).encode(), + ) + time.sleep(0.5) + # Should not crash; confirms message gets routed without error. + + # ------------------------------------------------------------------ + # Shut down the subscriber by cancelling its task. + # ------------------------------------------------------------------ + if sub_task_holder and sub_loop: + sub_loop[0].call_soon_threadsafe(sub_task_holder[0].cancel) + sub_thread.join(timeout=5.0) + + settings_mod.reset_for_tests() diff --git a/tests/test_mqtt_events.py b/tests/test_mqtt_events.py new file mode 100644 index 0000000..a2dcf34 --- /dev/null +++ b/tests/test_mqtt_events.py @@ -0,0 +1,153 @@ +"""Regression tests for MQTT event emission (issues 1-2). + +Issue 1: emit_queue_changed broadcasts queue state counts. +Issue 2: scanner.scan emits clip_indexed after indexing. +""" +from __future__ import annotations + +import asyncio +import time + +import pytest + + +# --------------------------------------------------------------------------- +# Issue 1: emit_queue_changed +# --------------------------------------------------------------------------- + +def _make_db(tmp_path): + from web.db import Database + return Database(str(tmp_path / "v.db")) + + +class _FakeHub: + """Minimal hub stand-in that records broadcast calls.""" + + def __init__(self): + self.broadcasts: list[dict] = [] + self.scheduled: list[dict] = [] + self.last_state: dict = {} + + async def broadcast(self, event: dict) -> None: + self.broadcasts.append(event) + + def schedule_broadcast(self, loop, event: dict) -> None: + self.scheduled.append(event) + + +@pytest.mark.asyncio +async def test_emit_queue_changed_from_async_context(tmp_path): + """emit_queue_changed uses create_task when there is a running loop.""" + from web.services.queue import emit_queue_changed + + db = _make_db(tmp_path) + hub = _FakeHub() + + now = int(time.time()) + with db.write() as c: + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at) VALUES (?,?,?,?)", + ("a.MP4", "/DCIM", "pending", now), + ) + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at) VALUES (?,?,?,?)", + ("b.MP4", "/DCIM", "failed", now), + ) + + emit_queue_changed(db, hub) + # give the event loop a turn to run the created task + await asyncio.sleep(0) + + assert len(hub.broadcasts) == 1 + ev = hub.broadcasts[0] + assert ev["type"] == "queue_changed" + assert ev["pending"] == 1 + assert ev["failed"] == 1 + assert ev["downloading"] == 0 + + +def test_emit_queue_changed_noop_when_hub_none(tmp_path): + """emit_queue_changed is a no-op when hub is None (safe default).""" + from web.services.queue import emit_queue_changed + db = _make_db(tmp_path) + # Should not raise. + emit_queue_changed(db, None) + + +def test_emit_queue_changed_uses_schedule_broadcast_from_thread(tmp_path): + """When there's no running loop and loop is passed, schedule_broadcast fires.""" + from web.services.queue import emit_queue_changed + + db = _make_db(tmp_path) + hub = _FakeHub() + + # Run emit_queue_changed from a plain thread context (no running loop). + # We simulate this by calling it inside asyncio.run's shutdown gap; + # the simplest approach is to use a fresh thread. + result: list[dict] = [] + + def _in_thread(): + # Allocate a loop but don't set it as running — mirrors executor thread. + loop = asyncio.new_event_loop() + try: + hub_local = _FakeHub() + emit_queue_changed(db, hub_local, loop=loop) + result.extend(hub_local.scheduled) + finally: + loop.close() + + import threading + t = threading.Thread(target=_in_thread) + t.start() + t.join(timeout=3.0) + + assert len(result) == 1 + assert result[0]["type"] == "queue_changed" + + +# --------------------------------------------------------------------------- +# Issue 2: scanner.scan emits clip_indexed +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_scan_emits_clip_indexed(tmp_path): + """scanner.scan schedules clip_indexed after indexing (called from thread).""" + from web.services import scanner + from web.db import Database + + db = Database(str(tmp_path / "v.db")) + hub = _FakeHub() + + # scan over an empty directory — total will be 0 but event must fire. + # asyncio.to_thread runs scan on an executor thread, so get_running_loop() + # raises inside scan — it falls back to schedule_broadcast with the loop. + loop = asyncio.get_running_loop() + n = await asyncio.to_thread( + scanner.scan, db, str(tmp_path), "daily", hub, loop, + ) + # give the loop a turn to drain any scheduled coroutines + await asyncio.sleep(0) + + assert n == 0 + # scan runs on a thread → schedule_broadcast path is taken + assert len(hub.scheduled) == 1 + ev = hub.scheduled[0] + assert ev["type"] == "clip_indexed" + assert ev["total"] == 0 + + +@pytest.mark.asyncio +async def test_scan_no_event_when_hub_none(tmp_path): + """scanner.scan with hub=None doesn't crash and returns count normally.""" + from web.services import scanner + from web.db import Database + + db = Database(str(tmp_path / "v.db")) + n = await asyncio.to_thread( + scanner.scan, db, str(tmp_path), "daily", + ) + assert n == 0 # empty dir + + diff --git a/tests/test_mqtt_hub_bridge.py b/tests/test_mqtt_hub_bridge.py new file mode 100644 index 0000000..a57651e --- /dev/null +++ b/tests/test_mqtt_hub_bridge.py @@ -0,0 +1,21 @@ +"""Hub bridge: map a Hub event type to the entities it affects.""" +from __future__ import annotations + + +def test_event_type_to_entities(): + from web.services.mqtt import entities_affected_by + affected = entities_affected_by("sync_state") + obj_ids = {e.object_id for e in affected} + assert "sync_status" in obj_ids + + +def test_clip_indexed_affects_archive_entities(): + from web.services.mqtt import entities_affected_by + obj_ids = {e.object_id for e in entities_affected_by("clip_indexed")} + assert "last_downloaded_clip" in obj_ids + assert "total_clips" in obj_ids + + +def test_unknown_event_yields_empty(): + from web.services.mqtt import entities_affected_by + assert list(entities_affected_by("this_event_does_not_exist")) == [] diff --git a/tests/test_mqtt_publication.py b/tests/test_mqtt_publication.py new file mode 100644 index 0000000..3273a48 --- /dev/null +++ b/tests/test_mqtt_publication.py @@ -0,0 +1,119 @@ +"""Publication logic: change detection + coalescing.""" +from __future__ import annotations + +import asyncio + +import pytest + + +@pytest.mark.asyncio +async def test_first_publish_emitted(monkeypatch): + from web.services.mqtt import PublishCoalescer + pc = PublishCoalescer(monotonic=lambda: 0.0) + sent = [] + async def sink(topic, payload, retain, qos): + sent.append((topic, payload, retain, qos)) + await pc.consider("a/state", b"42", min_interval=1.0, sink=sink, + retain=True, qos=1) + assert sent == [("a/state", b"42", True, 1)] + + +@pytest.mark.asyncio +async def test_unchanged_payload_suppressed(monkeypatch): + from web.services.mqtt import PublishCoalescer + now = [0.0] + pc = PublishCoalescer(monotonic=lambda: now[0]) + sent = [] + async def sink(topic, payload, retain, qos): + sent.append((topic, payload)) + await pc.consider("a/state", b"42", min_interval=0.0, + sink=sink, retain=True, qos=1) + now[0] = 10.0 # well past any interval + await pc.consider("a/state", b"42", min_interval=0.0, + sink=sink, retain=True, qos=1) + assert sent == [("a/state", b"42")] + + +@pytest.mark.asyncio +async def test_changed_payload_emitted_after_interval(monkeypatch): + from web.services.mqtt import PublishCoalescer + now = [0.0] + pc = PublishCoalescer(monotonic=lambda: now[0]) + sent = [] + async def sink(topic, payload, retain, qos): + sent.append((topic, payload)) + + await pc.consider("a/state", b"1", min_interval=2.0, + sink=sink, retain=True, qos=1) + now[0] = 0.5 + await pc.consider("a/state", b"2", min_interval=2.0, + sink=sink, retain=True, qos=1) + # Within the interval — should NOT have fired the second publish yet. + # But the value is now pending. + assert sent == [("a/state", b"1")] + + # When the interval elapses, the deadline-flush yields the latest value. + now[0] = 2.5 + await pc.flush_due(sink) + assert sent == [("a/state", b"1"), ("a/state", b"2")] + + +@pytest.mark.asyncio +async def test_intermediate_frames_dropped(monkeypatch): + from web.services.mqtt import PublishCoalescer + now = [0.0] + pc = PublishCoalescer(monotonic=lambda: now[0]) + sent = [] + async def sink(topic, payload, retain, qos): + sent.append(payload) + + await pc.consider("a", b"1", min_interval=5.0, + sink=sink, retain=False, qos=1) + now[0] = 1.0 + await pc.consider("a", b"2", min_interval=5.0, + sink=sink, retain=False, qos=1) + now[0] = 2.0 + await pc.consider("a", b"3", min_interval=5.0, + sink=sink, retain=False, qos=1) + now[0] = 6.0 + await pc.flush_due(sink) + # Only the first and the final value should have been sent. + assert sent == [b"1", b"3"] + + +@pytest.mark.asyncio +async def test_flush_due_does_nothing_when_no_pending(monkeypatch): + from web.services.mqtt import PublishCoalescer + pc = PublishCoalescer(monotonic=lambda: 0.0) + sent = [] + async def sink(*a, **kw): + sent.append(1) + await pc.flush_due(sink) + assert sent == [] + + +@pytest.mark.asyncio +async def test_revert_to_published_value_cancels_pending(): + """If a stashed-pending value is overwritten by the originally-published + value, the pending entry should be cancelled (no redundant publish on + flush_due).""" + from web.services.mqtt import PublishCoalescer + now = [0.0] + pc = PublishCoalescer(monotonic=lambda: now[0]) + sent = [] + async def sink(topic, payload, retain, qos): + sent.append(payload) + + await pc.consider("a", b"1", min_interval=5.0, + sink=sink, retain=False, qos=1) + now[0] = 1.0 + await pc.consider("a", b"2", min_interval=5.0, + sink=sink, retain=False, qos=1) + now[0] = 2.0 + # Revert to the originally-published value while still inside the + # cooldown — should cancel pending and emit nothing. + await pc.consider("a", b"1", min_interval=5.0, + sink=sink, retain=False, qos=1) + now[0] = 6.0 + await pc.flush_due(sink) + assert sent == [b"1"] # only the original publish diff --git a/tests/test_mqtt_settings.py b/tests/test_mqtt_settings.py new file mode 100644 index 0000000..fb7ee7c --- /dev/null +++ b/tests/test_mqtt_settings.py @@ -0,0 +1,97 @@ +"""Tests for MQTT_* settings.""" +from __future__ import annotations + +import pytest + + +def test_defaults(): + from web.settings_schema import DEFAULT_VALUES + assert DEFAULT_VALUES["MQTT_ENABLED"] is False + assert DEFAULT_VALUES["MQTT_HOST"] == "" + assert DEFAULT_VALUES["MQTT_PORT"] == 1883 + assert DEFAULT_VALUES["MQTT_USERNAME"] == "" + assert DEFAULT_VALUES["MQTT_PASSWORD"] == "" + assert DEFAULT_VALUES["MQTT_TLS"] is False + assert DEFAULT_VALUES["MQTT_CLIENT_ID"] == "" + assert DEFAULT_VALUES["MQTT_DISCOVERY_PREFIX"] == "homeassistant" + assert DEFAULT_VALUES["MQTT_NODE_ID"] == "viofosync" + assert DEFAULT_VALUES["MQTT_DISCOVERY_ENABLED"] is True + assert DEFAULT_VALUES["MQTT_QOS"] == 1 + + +def test_all_editable(): + from web.settings_schema import EDITABLE_KEYS + for k in ( + "MQTT_ENABLED", "MQTT_HOST", "MQTT_PORT", "MQTT_USERNAME", + "MQTT_PASSWORD", "MQTT_TLS", "MQTT_CLIENT_ID", + "MQTT_DISCOVERY_PREFIX", "MQTT_NODE_ID", + "MQTT_DISCOVERY_ENABLED", "MQTT_QOS", + ): + assert k in EDITABLE_KEYS + + +def test_port_validation(): + from web.settings_schema import validate_partial + with pytest.raises(ValueError): + validate_partial({"MQTT_PORT": 0}) + with pytest.raises(ValueError): + validate_partial({"MQTT_PORT": 70000}) + assert validate_partial({"MQTT_PORT": 8883})["MQTT_PORT"] == 8883 + + +def test_node_id_charset(): + from web.settings_schema import validate_partial + validate_partial({"MQTT_NODE_ID": "viofosync_garage_2"}) + with pytest.raises(ValueError): + validate_partial({"MQTT_NODE_ID": "Viofosync"}) # uppercase + with pytest.raises(ValueError): + validate_partial({"MQTT_NODE_ID": "viofosync-garage"}) # hyphen + with pytest.raises(ValueError): + validate_partial({"MQTT_NODE_ID": ""}) + + +def test_discovery_prefix_no_slashes(): + from web.settings_schema import validate_partial + validate_partial({"MQTT_DISCOVERY_PREFIX": "homeassistant"}) + with pytest.raises(ValueError): + validate_partial({"MQTT_DISCOVERY_PREFIX": "/homeassistant"}) + with pytest.raises(ValueError): + validate_partial({"MQTT_DISCOVERY_PREFIX": "homeassistant/"}) + + +def test_qos_literal(): + from web.settings_schema import validate_partial + for v in (0, 1, 2): + assert validate_partial({"MQTT_QOS": v})["MQTT_QOS"] == v + with pytest.raises(ValueError): + validate_partial({"MQTT_QOS": 3}) + + +def test_host_required_when_enabled(tmp_config_dir): + """Cross-field rule: cannot enable MQTT without a host.""" + from web import settings as settings_mod + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + with pytest.raises(ValueError): + p.update({"MQTT_ENABLED": True}, actor="test") + # Setting both at once works: + p.update({"MQTT_ENABLED": True, "MQTT_HOST": "broker.lan"}, actor="test") + snap = p.get() + assert snap.mqtt_enabled is True + assert snap.mqtt_host == "broker.lan" + settings_mod.reset_for_tests() + + +def test_snapshot_projection(tmp_config_dir): + """The Snapshot dataclass exposes every MQTT_* key as a lower-snake-case attr.""" + from web import settings as settings_mod + settings_mod.reset_for_tests() + snap = settings_mod.get_provider().get() + for attr in ( + "mqtt_enabled", "mqtt_host", "mqtt_port", "mqtt_username", + "mqtt_password", "mqtt_tls", "mqtt_client_id", + "mqtt_discovery_prefix", "mqtt_node_id", + "mqtt_discovery_enabled", "mqtt_qos", + ): + assert hasattr(snap, attr), attr + settings_mod.reset_for_tests() diff --git a/tests/test_mqtt_settings_reload.py b/tests/test_mqtt_settings_reload.py new file mode 100644 index 0000000..537d5c8 --- /dev/null +++ b/tests/test_mqtt_settings_reload.py @@ -0,0 +1,85 @@ +"""Tests for MqttService.on_settings_changed.""" +from __future__ import annotations + +import asyncio + +import pytest + + +@pytest.mark.asyncio +async def test_on_settings_changed_restarts_on_connection_keys(monkeypatch): + from web.services.mqtt import MqttService + + started = [] + stopped = [] + + class S(MqttService): + def start(self): + started.append(1) + async def stop(self): + stopped.append(1) + + svc = S(db=None, provider=None, hub=None, app=None) + # Stub provider snapshot + svc._provider = type("P", (), { + "get": staticmethod(lambda: type("Snap", (), { + "mqtt_enabled": True, "mqtt_host": "h", + })()), + })() + await svc.on_settings_changed({"MQTT_HOST"}, svc._provider.get()) + assert started and stopped + + +@pytest.mark.asyncio +async def test_on_settings_changed_skips_irrelevant_keys(): + from web.services.mqtt import MqttService + started = [] + + class S(MqttService): + def start(self): + started.append(1) + async def stop(self): + return + + svc = S(db=None, provider=None, hub=None, app=None) + svc._provider = type("P", (), { + "get": staticmethod(lambda: type("Snap", (), { + "mqtt_enabled": True, "mqtt_host": "h", + })()), + })() + await svc.on_settings_changed({"ADDRESS"}, svc._provider.get()) + assert started == [] + + +@pytest.mark.asyncio +async def test_on_settings_changed_node_rename_emits_cleanup_publishes(): + """When MQTT_NODE_ID changes, the service publishes empty payloads + to every old discovery topic before restarting.""" + from web.services.mqtt import MqttService + + cleared: list[str] = [] + + class S(MqttService): + def start(self): pass + async def stop(self): pass + async def _publish_now(self, topic, payload, retain, qos): + cleared.append((topic, payload, retain)) + + svc = S(db=None, provider=None, hub=None, app=None) + svc._last_node_id = "viofosync" + svc._last_discovery_prefix = "homeassistant" + + new_snap = type("Snap", (), { + "mqtt_enabled": True, "mqtt_host": "h", + "mqtt_node_id": "viofosync_garage", + "mqtt_discovery_prefix": "homeassistant", + })() + svc._provider = type("P", (), {"get": staticmethod(lambda: new_snap)})() + await svc.on_settings_changed({"MQTT_NODE_ID"}, new_snap) + + assert any("homeassistant/sensor/viofosync/queue_pending/config" in t + for (t, _p, _r) in cleared) + # Empty payload = HA delete signal + for _t, p, retain in cleared: + assert p == b"" + assert retain is True diff --git a/tests/test_mqtt_state_extraction.py b/tests/test_mqtt_state_extraction.py new file mode 100644 index 0000000..1052f76 --- /dev/null +++ b/tests/test_mqtt_state_extraction.py @@ -0,0 +1,314 @@ +"""State-extraction tests. Each state_fn is a pure function over +(hub, db, snapshot) so they're easy to exercise without a broker.""" +from __future__ import annotations + +import time +import types + +import pytest + + +def _stub_snapshot(**kwargs): + """Make a stub Snapshot. We don't need every field — the state + fns only touch a subset.""" + base = dict( + address="192.168.1.50", + recordings=".", + enable_scheduled_sync=True, + retention_max_days=0, + ) + base.update(kwargs) + return types.SimpleNamespace(**base) + + +def _hub_with_state(state: dict): + return types.SimpleNamespace(last_state=state) + + +def _db_with_clip_index(tmp_path, rows: list[dict]): + from web.db import Database + db = Database(str(tmp_path / "v.db")) + with db.write() as c: + for r in rows: + c.execute( + "INSERT INTO clip_index " + "(path, basename, group_name, timestamp, camera, " + " sequence, event_type, size_bytes, has_gpx, " + " gps_examined, scanned_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (r["path"], r["basename"], r["group_name"], r["timestamp"], + r["camera"], r["sequence"], r["event_type"], r["size_bytes"], + 0, 0, r["timestamp"]), + ) + return db + + +def _db_with_queue(tmp_path, rows: list[tuple[str, str]]): + """rows: list of (filename, state).""" + from web.db import Database + db = Database(str(tmp_path / "v.db")) + now = int(time.time()) + with db.write() as c: + for (filename, state) in rows: + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at) " + "VALUES (?, ?, ?, ?)", + (filename, "/DCIM/Movie", state, now), + ) + return db + + +# ---- binary sensors + +def test_state_dashcam_online(): + from web.services.mqtt_state import state_dashcam + assert state_dashcam(_hub_with_state({"dashcam_online": True}), + None, _stub_snapshot()) == "ON" + assert state_dashcam(_hub_with_state({"dashcam_online": False}), + None, _stub_snapshot()) == "OFF" + + +def test_state_dashcam_unknown_when_no_address(): + from web.services.mqtt_state import state_dashcam + # No address configured → reachable is meaningless + assert state_dashcam(_hub_with_state({"dashcam_online": True}), + None, _stub_snapshot(address="")) == "OFF" + + +def test_state_sync_status_stopped_when_missing(): + from web.services.mqtt_state import state_sync_status + # sync_state key absent → stopped + hub = _hub_with_state({}) + assert state_sync_status(hub, None, _stub_snapshot()) == "stopped" + + +def test_state_sync_status_stopped_when_not_running(): + from web.services.mqtt_state import state_sync_status + hub = _hub_with_state({"sync_state": {"running": False, "paused": False}}) + assert state_sync_status(hub, None, _stub_snapshot()) == "stopped" + + +def test_state_sync_status_paused(): + from web.services.mqtt_state import state_sync_status + hub = _hub_with_state({"sync_state": {"running": True, "paused": True}}) + assert state_sync_status(hub, None, _stub_snapshot()) == "paused" + + +def test_state_sync_status_downloading(): + from web.services.mqtt_state import state_sync_status + hub = _hub_with_state({ + "sync_state": {"running": True, "paused": False}, + "current_item": {"filename": "x.MP4"}, + }) + assert state_sync_status(hub, None, _stub_snapshot()) == "downloading" + + +def test_state_sync_status_idle_when_queue_empty(tmp_path): + """No current item AND no pending/downloading rows → idle.""" + from web.services.mqtt_state import state_sync_status + db = _db_with_queue(tmp_path, []) + hub = _hub_with_state({ + "sync_state": {"running": True, "paused": False}, + "current_item": None, + }) + assert state_sync_status(hub, db, _stub_snapshot()) == "idle" + + +def test_state_sync_status_downloading_between_items(tmp_path): + """No current item but queue still has pending rows → downloading + (covers the gap between consecutive clips in a sync cycle).""" + from web.services.mqtt_state import state_sync_status + db = _db_with_queue(tmp_path, [("next.MP4", "pending")]) + hub = _hub_with_state({ + "sync_state": {"running": True, "paused": False}, + "current_item": None, + }) + assert state_sync_status(hub, db, _stub_snapshot()) == "downloading" + + +def test_state_sync_status_downloading_while_row_marked_downloading(tmp_path): + """A row in state='downloading' also counts as work in progress + even if hub.last_state hasn't seen item_started yet.""" + from web.services.mqtt_state import state_sync_status + db = _db_with_queue(tmp_path, [("x.MP4", "downloading")]) + hub = _hub_with_state({ + "sync_state": {"running": True, "paused": False}, + "current_item": None, + }) + assert state_sync_status(hub, db, _stub_snapshot()) == "downloading" + + +def test_state_sync_status_idle_ignores_done_and_failed(tmp_path): + """Failed/done rows don't drag status back to downloading.""" + from web.services.mqtt_state import state_sync_status + db = _db_with_queue(tmp_path, [ + ("done.MP4", "done"), + ("fail.MP4", "failed"), + ("gone.MP4", "gone"), + ]) + hub = _hub_with_state({ + "sync_state": {"running": True, "paused": False}, + "current_item": None, + }) + assert state_sync_status(hub, db, _stub_snapshot()) == "idle" + + +# ---- queue counts + +def test_state_queue_pending(tmp_path): + from web.services.mqtt_state import state_queue_pending + db = _db_with_queue(tmp_path, [ + ("a.MP4", "pending"), + ("b.MP4", "pending"), + ("c.MP4", "failed"), + ]) + assert state_queue_pending(_hub_with_state({}), db, + _stub_snapshot()) == "2" + + +def test_state_queue_failed(tmp_path): + from web.services.mqtt_state import state_queue_failed + db = _db_with_queue(tmp_path, [ + ("a.MP4", "pending"), + ("b.MP4", "failed"), + ]) + assert state_queue_failed(_hub_with_state({}), db, + _stub_snapshot()) == "1" + + +def test_state_queue_downloading(tmp_path): + from web.services.mqtt_state import state_queue_downloading + db = _db_with_queue(tmp_path, [ + ("a.MP4", "downloading"), + ]) + assert state_queue_downloading(_hub_with_state({}), db, + _stub_snapshot()) == "1" + + +# ---- archive + +def test_state_last_downloaded_clip_returns_iso(tmp_path): + from web.services.mqtt_state import state_last_downloaded_clip + ts = 1715852400 # 2024-05-16 ish + db = _db_with_clip_index(tmp_path, [{ + "path": "/r/a.MP4", "basename": "a.MP4", + "group_name": "2024-05-16", + "timestamp": ts, "camera": "F", "sequence": 1, + "event_type": "normal", "size_bytes": 1024, + }]) + out = state_last_downloaded_clip(_hub_with_state({}), db, _stub_snapshot()) + assert out is not None + # ISO 8601 with 'Z' suffix for HA timestamp device_class + assert out.endswith("Z") or "+" in out + + +def test_state_last_downloaded_clip_none_when_empty(tmp_path): + from web.services.mqtt_state import state_last_downloaded_clip + db = _db_with_clip_index(tmp_path, []) + assert state_last_downloaded_clip(_hub_with_state({}), db, + _stub_snapshot()) is None + + +def test_state_total_clips(tmp_path): + from web.services.mqtt_state import state_total_clips + db = _db_with_clip_index(tmp_path, [ + {"path": "/r/a.MP4", "basename": "a.MP4", + "group_name": "d", "timestamp": 1, "camera": "F", "sequence": 1, + "event_type": "normal", "size_bytes": 0}, + {"path": "/r/b.MP4", "basename": "b.MP4", + "group_name": "d", "timestamp": 2, "camera": "R", "sequence": 1, + "event_type": "normal", "size_bytes": 0}, + ]) + assert state_total_clips(_hub_with_state({}), db, + _stub_snapshot()) == "2" + + +# ---- current download + +def test_state_current_filename(): + from web.services.mqtt_state import state_current_filename + hub = _hub_with_state({"current_item": {"filename": "2026_0516_120000_001F.MP4"}}) + assert state_current_filename(hub, None, + _stub_snapshot()) == "2026_0516_120000_001F.MP4" + assert state_current_filename(_hub_with_state({"current_item": None}), + None, _stub_snapshot()) is None + + +def test_state_current_progress_pct(): + from web.services.mqtt_state import state_current_progress + hub = _hub_with_state({ + "current_item": {"filename": "x", "bytes": 50, "total": 100}, + }) + assert state_current_progress(hub, None, _stub_snapshot()) == "50.0" + # No total = no progress + hub = _hub_with_state({"current_item": {"filename": "x", "bytes": 50}}) + assert state_current_progress(hub, None, _stub_snapshot()) is None + + +# ---- disk + +def test_state_disk_used_filesystem_only(tmp_path): + """No quota set → just the filesystem percentage.""" + from web.services.mqtt_state import state_disk_used + out = state_disk_used( + _hub_with_state({}), None, + _stub_snapshot(recordings=str(tmp_path), recordings_quota_gb=0), + ) + assert out is not None + val = int(out) + assert 0 <= val <= 100 + + +def test_state_disk_used_reports_max_of_quota_and_filesystem(tmp_path): + """Both rules active → publish the higher percentage (the rule + closest to triggering cleanup).""" + from web.services.mqtt_state import state_disk_used + from web.services import retention as _ret + + _ret._size_cache.clear() + # Plant exactly 600 MiB under recordings, then set a tiny 1 GiB quota. + # Quota % = 600/1024 ≈ 58.6%. Filesystem % on the test runner is + # almost certainly much lower than that, so max should be ≈59. + for i in range(600): + (tmp_path / f"chunk_{i}.MP4").write_bytes(b"\0" * (1 << 20)) + out = state_disk_used( + _hub_with_state({}), None, + _stub_snapshot(recordings=str(tmp_path), recordings_quota_gb=1), + ) + assert out is not None + pct = int(out) + assert pct >= 58, f"expected the quota rule (~59%) to dominate, got {pct}%" + + +def test_state_disk_used_filesystem_wins_when_quota_far_from_full(tmp_path): + """When the quota is generous and the filesystem is the tighter + constraint, the FS % wins.""" + from web.services.mqtt_state import state_disk_used + from web.services import retention as _ret + + _ret._size_cache.clear() + # 1 MiB of data under recordings, 1024 GiB quota → 0% quota usage. + # The filesystem on the test runner is going to be busier than that, + # so the FS rule wins and the sensor reports the FS %. + (tmp_path / "tiny.MP4").write_bytes(b"\0" * (1 << 20)) + out = state_disk_used( + _hub_with_state({}), None, + _stub_snapshot(recordings=str(tmp_path), recordings_quota_gb=1024), + ) + assert out is not None + # Quota would have reported ~0; max(0, fs%) ≈ fs%, almost always > 0. + pct = int(out) + assert 0 <= pct <= 100 + + +def test_state_disk_used_missing_path_returns_none(tmp_path): + from web.services.mqtt_state import state_disk_used + out = state_disk_used( + _hub_with_state({}), None, + _stub_snapshot(recordings=str(tmp_path / "does-not-exist"), + recordings_quota_gb=0), + ) + assert out is None + + diff --git a/tests/test_mqtt_status.py b/tests/test_mqtt_status.py new file mode 100644 index 0000000..93b9365 --- /dev/null +++ b/tests/test_mqtt_status.py @@ -0,0 +1,77 @@ +"""Status reporting on MqttService (in-process, no broker).""" +from __future__ import annotations + + +def test_initial_status_idle(): + from web.services.mqtt import MqttService, ConnState + svc = MqttService(db=None, provider=None, hub=None, app=None) + s = svc.get_status() + assert s["state"] == ConnState.IDLE.value + assert s["detail"] is None + + +def test_status_after_marking_connected(): + from web.services.mqtt import MqttService, ConnState + svc = MqttService(db=None, provider=None, hub=None, app=None) + svc._set_state(ConnState.CONNECTED, detail="broker:1883") + s = svc.get_status() + assert s["state"] == "connected" + assert s["detail"] == "broker:1883" + + +def test_status_after_marking_error(): + from web.services.mqtt import MqttService, ConnState + svc = MqttService(db=None, provider=None, hub=None, app=None) + svc._set_state(ConnState.ERROR, detail="auth failed") + s = svc.get_status() + assert s["state"] == "error" + assert s["detail"] == "auth failed" + + +def test_status_endpoint_requires_session(tmp_config_dir, tmp_recordings_dir, + monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load(); data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + app = create_app() + with TestClient(app) as c: + # No session → 401 + r = c.get("/api/mqtt/status") + assert r.status_code == 401 + + +def test_status_endpoint_returns_idle_when_disabled( + tmp_config_dir, tmp_recordings_dir, monkeypatch, +): + import bcrypt + from fastapi.testclient import TestClient + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load(); data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + app = create_app() + with TestClient(app) as c: + # login + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + r = c.get("/api/mqtt/status") + assert r.status_code == 200 + body = r.json() + # MQTT defaults to disabled, so state is idle + assert body["state"] in ("idle", "disabled") diff --git a/tests/test_mqtt_test_endpoint.py b/tests/test_mqtt_test_endpoint.py new file mode 100644 index 0000000..07763c5 --- /dev/null +++ b/tests/test_mqtt_test_endpoint.py @@ -0,0 +1,73 @@ +"""Test for POST /api/mqtt/test.""" +from __future__ import annotations + +import pytest + + +@pytest.fixture +def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load(); data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + csrf = c.get("/api/auth/csrf").json()["csrf"] + c.headers["X-CSRF-Token"] = csrf + yield c + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def test_test_endpoint_validates_host(logged_in_client): + r = logged_in_client.post( + "/api/mqtt/test", + json={"host": "", "port": 1883}, + ) + assert r.status_code == 400 + + +def test_test_endpoint_returns_failure_for_unreachable_host(logged_in_client, monkeypatch): + # Monkeypatch aiomqtt.Client to raise on connect. + import aiomqtt + + class _Boom: + def __init__(self, **_kw): pass + async def __aenter__(self): raise aiomqtt.MqttError("connection refused") + async def __aexit__(self, *a): pass + monkeypatch.setattr(aiomqtt, "Client", _Boom) + r = logged_in_client.post( + "/api/mqtt/test", + json={"host": "127.0.0.1", "port": 1}, + ) + assert r.status_code == 200 + body = r.json() + assert body["ok"] is False + assert "connection" in body["detail"].lower() + + +def test_test_endpoint_returns_success(logged_in_client, monkeypatch): + import aiomqtt + + class _Ok: + def __init__(self, **_kw): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): pass + monkeypatch.setattr(aiomqtt, "Client", _Ok) + r = logged_in_client.post( + "/api/mqtt/test", + json={"host": "h", "port": 1883}, + ) + assert r.status_code == 200 + assert r.json()["ok"] is True diff --git a/tests/test_mqtt_topology.py b/tests/test_mqtt_topology.py new file mode 100644 index 0000000..29a0549 --- /dev/null +++ b/tests/test_mqtt_topology.py @@ -0,0 +1,95 @@ +"""Topology integrity tests.""" +from __future__ import annotations + + +def test_topology_has_expected_entities(): + from web.services.mqtt_topology import TOPOLOGY + obj_ids = {e.object_id for e in TOPOLOGY} + expected = { + # binary_sensors / sensors (Task 5) + "dashcam", "sync_status", + "queue_pending", "queue_failed", "queue_downloading", + "last_downloaded_clip", "total_clips", "current_filename", + "current_progress", "disk_used", + } + assert expected.issubset(obj_ids), expected - obj_ids + + +def test_unique_ids_unique_per_node(): + from web.services.mqtt_topology import ( + TOPOLOGY, build_unique_id, + ) + cfg = {"discovery_prefix": "homeassistant", "node_id": "viofosync", + "version": "0.2.0"} + uids = [build_unique_id(e.object_id, cfg) for e in TOPOLOGY] + assert len(uids) == len(set(uids)) + + +def test_sensors_have_state_fn(): + from web.services.mqtt_topology import TOPOLOGY + for e in TOPOLOGY: + if e.component in ("sensor", "binary_sensor"): + assert e.state_fn is not None, e.object_id + + +def test_default_enabled_set(): + from web.services.mqtt_topology import TOPOLOGY + enabled_by_default = { + e.object_id for e in TOPOLOGY if e.enabled_by_default + } + assert { + "dashcam", "sync_status", + "queue_pending", "last_downloaded_clip", "disk_used", + }.issubset(enabled_by_default) + # The verbose ones are off by default + disabled = { + e.object_id for e in TOPOLOGY if not e.enabled_by_default + } + assert { + "queue_failed", "queue_downloading", "current_filename", + "current_progress", "total_clips", + }.issubset(disabled) + + +def test_publish_intervals(): + """Coalescing intervals match the spec's table.""" + from web.services.mqtt_topology import TOPOLOGY + by_id = {e.object_id: e for e in TOPOLOGY} + assert by_id["current_filename"].min_publish_interval_s == 2.0 + assert by_id["current_progress"].min_publish_interval_s == 2.0 + assert by_id["queue_pending"].min_publish_interval_s == 1.0 + assert by_id["queue_failed"].min_publish_interval_s == 1.0 + assert by_id["queue_downloading"].min_publish_interval_s == 1.0 + assert by_id["last_downloaded_clip"].min_publish_interval_s == 5.0 + assert by_id["total_clips"].min_publish_interval_s == 5.0 + + +def test_button_entries_present(): + from web.services.mqtt_topology import TOPOLOGY + button_ids = {e.object_id for e in TOPOLOGY + if e.component == "button"} + assert button_ids == { + "start_sync", "pause_sync", "skip_current", + "refresh_queue", "retry_failed", "rescan_archive", + } + + +def test_buttons_have_no_state_fn(): + from web.services.mqtt_topology import TOPOLOGY + for e in TOPOLOGY: + if e.component == "button": + assert e.state_fn is None, e.object_id + + +def test_button_default_enabled(): + from web.services.mqtt_topology import TOPOLOGY + for e in TOPOLOGY: + if e.component == "button": + assert e.enabled_by_default is True, e.object_id + + +def test_command_handler_present_only_on_buttons(): + from web.services.mqtt_topology import TOPOLOGY + for e in TOPOLOGY: + if e.command_handler is not None: + assert e.component == "button", e.object_id diff --git a/tests/test_queue_retry_failed.py b/tests/test_queue_retry_failed.py new file mode 100644 index 0000000..678b53f --- /dev/null +++ b/tests/test_queue_retry_failed.py @@ -0,0 +1,41 @@ +"""Tests for queue.retry_failed.""" +from __future__ import annotations + +import time + + +def test_retry_failed_resets_state_and_attempts(tmp_path): + from web.db import Database + from web.services.queue import retry_failed + db = Database(str(tmp_path / "v.db")) + now = int(time.time()) + with db.write() as c: + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at, attempts, last_error) " + "VALUES (?, ?, ?, ?, ?, ?)", + ("a.MP4", "/DCIM", "failed", now, 3, "boom"), + ) + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, enqueued_at, attempts) " + "VALUES (?, ?, ?, ?, ?)", + ("b.MP4", "/DCIM", "pending", now, 0), + ) + n = retry_failed(db) + assert n == 1 + with db.conn() as c: + rows = {r["filename"]: dict(r) for r in c.execute( + "SELECT * FROM download_queue" + ).fetchall()} + assert rows["a.MP4"]["state"] == "pending" + assert rows["a.MP4"]["attempts"] == 0 + assert rows["a.MP4"]["last_error"] is None + assert rows["b.MP4"]["state"] == "pending" # untouched + + +def test_retry_failed_noop_when_no_failed_rows(tmp_path): + from web.db import Database + from web.services.queue import retry_failed + db = Database(str(tmp_path / "v.db")) + assert retry_failed(db) == 0 diff --git a/tests/test_storage_endpoint.py b/tests/test_storage_endpoint.py index 8f8c051..933f72b 100644 --- a/tests/test_storage_endpoint.py +++ b/tests/test_storage_endpoint.py @@ -4,6 +4,23 @@ import pytest +class _FakeMqttService: + """Stand-in so the storage tests don't carry MQTT side effects. + The real service's settings-change subscriber schedules an async + task that survives past the TestClient context and leaks the + coroutine into the next test's setup phase.""" + + def __init__(self, **kwargs): + self._last_node_id = "" + self._last_discovery_prefix = "" + + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): + return {"state": "idle", "detail": None, "last_published_at": None} + + @pytest.fixture def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): import bcrypt @@ -22,6 +39,7 @@ def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): settings_mod.reset_for_tests() monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) app = create_app() c = TestClient(app) @@ -72,6 +90,7 @@ def test_usage_quota_mode_reports_against_declared_quota( from web import settings as settings_mod from web.services import retention + # Plant ~2 MiB under recordings, then set a 1 GiB quota. rec = tmp_recordings_dir (rec / "clip.MP4").write_bytes(b"\0" * (2 << 20)) retention._size_cache.clear() @@ -79,6 +98,9 @@ def test_usage_quota_mode_reports_against_declared_quota( p = settings_mod.get_provider() p.update({"RECORDINGS_QUOTA_GB": 1}, actor="test") + # Need to fetch CSRF first because update() goes through a write path + # — actually, provider.update() doesn't need CSRF; only the HTTP PUT + # does. The settings change here is in-process. r = logged_in_client.get("/api/storage/usage") assert r.status_code == 200 body = r.json() @@ -111,6 +133,7 @@ def test_usage_includes_threshold_when_set( settings_mod.reset_for_tests() monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) with TestClient(create_app()) as c: c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) diff --git a/web/app.py b/web/app.py index d5f7730..041e81a 100644 --- a/web/app.py +++ b/web/app.py @@ -29,6 +29,7 @@ from .routers import progress as progress_router from .routers import queue as queue_router from .routers import settings as settings_router +from .routers import mqtt as mqtt_router from .routers import setup as setup_router from .routers import storage as storage_router from .services import retention as _ret_mod @@ -40,6 +41,7 @@ ) from .services.geocode import GeocodeService from .services.hub import Hub +from .services.mqtt import MqttService from .services.sync_worker import SyncWorker from .setup_mode import SetupModeMiddleware @@ -104,8 +106,10 @@ def _on_settings_changed( async def _background_scan() -> None: try: log.info("initial archive scan: starting (%s)", s.recordings) + loop = asyncio.get_running_loop() n = await asyncio.to_thread( scanner.scan, app.state.db, s.recordings, s.grouping, + app.state.hub, loop, ) log.info("initial archive scan: %d clips indexed", n) except Exception as e: # pragma: no cover — non-fatal @@ -172,10 +176,33 @@ async def _background_retention() -> None: "ADDRESS not set — sync worker idle until configured" ) + app.state.mqtt = MqttService( + db=app.state.db, + provider=provider, + hub=app.state.hub, + app=app, + ) + if s.mqtt_enabled and s.mqtt_host: + app.state.mqtt.start() + + # Track current discovery/node so on_settings_changed can publish + # cleanup deletes against the *old* topology when those change. + app.state.mqtt._last_node_id = s.mqtt_node_id + app.state.mqtt._last_discovery_prefix = s.mqtt_discovery_prefix + + def _on_mqtt_settings_change(keys, snap): + # Scheduled on the running loop so async work executes safely. + asyncio.create_task(app.state.mqtt.on_settings_changed(keys, snap)) + + provider.subscribe(_on_mqtt_settings_change) + try: yield finally: log.info("viofosync web UI shutting down") + mqtt_svc = getattr(app.state, "mqtt", None) + if mqtt_svc is not None: + await mqtt_svc.stop() for attr in ("initial_scan_task", "retention_task"): task = getattr(app.state, attr, None) if task is not None and not task.done(): @@ -196,7 +223,7 @@ def create_app() -> FastAPI: app = FastAPI( title="Viofosync", - version="0.1", + version="2.2", lifespan=lifespan, docs_url=None, # no swagger in prod build redoc_url=None, @@ -211,6 +238,7 @@ def create_app() -> FastAPI: app.include_router(progress_router.router) app.include_router(settings_router.router) app.include_router(setup_router.router) + app.include_router(mqtt_router.router) app.include_router(storage_router.router) # Static SPA — served at / with an explicit index.html fall-through diff --git a/web/routers/archive.py b/web/routers/archive.py index dc4cffa..a569f73 100644 --- a/web/routers/archive.py +++ b/web/routers/archive.py @@ -374,6 +374,7 @@ async def rescan(request: Request) -> JSONResponse: n = await asyncio.to_thread( scanner.scan, request.app.state.db, s.recordings, s.grouping, + request.app.state.hub, asyncio.get_running_loop(), ) asyncio.create_task( scanner.sweep_missing_thumbs( diff --git a/web/routers/mqtt.py b/web/routers/mqtt.py new file mode 100644 index 0000000..719eefe --- /dev/null +++ b/web/routers/mqtt.py @@ -0,0 +1,65 @@ +"""HTTP endpoints for the MQTT service: status + connection test. + +Both require an authenticated session; the test endpoint additionally +requires CSRF because it makes a one-shot outbound MQTT connection. +""" +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel + +from ..auth import require_csrf, require_session + + +router = APIRouter(prefix="/api/mqtt", tags=["mqtt"], + dependencies=[Depends(require_session)]) + + +@router.get("/status") +def get_status(request: Request) -> dict: + svc = getattr(request.app.state, "mqtt", None) + if svc is None: + return {"state": "idle", "detail": None, "last_published_at": None} + return svc.get_status() + + +class _TestBody(BaseModel): + host: str + port: int = 1883 + username: str = "" + password: str = "" + tls: bool = False + client_id: str = "" + + +@router.post("/test", dependencies=[Depends(require_csrf)]) +async def post_test(body: _TestBody) -> dict: + if not body.host: + raise HTTPException(400, "host is required") + import asyncio + import aiomqtt + import ssl as _ssl + + kwargs: dict = dict( + hostname=body.host, port=body.port, + username=body.username or None, + password=body.password or None, + identifier=body.client_id or None, + keepalive=10, + ) + if body.tls: + kwargs["tls_context"] = _ssl.create_default_context() + + async def _attempt() -> dict: + try: + async with aiomqtt.Client(**kwargs): + return {"ok": True, "detail": "connected"} + except aiomqtt.MqttError as e: + return {"ok": False, "detail": f"connection failed: {e}"} + except Exception as e: + return {"ok": False, "detail": f"error: {e}"} + + try: + return await asyncio.wait_for(_attempt(), timeout=5.0) + except asyncio.TimeoutError: + return {"ok": False, "detail": "connection timed out (5s)"} diff --git a/web/routers/queue.py b/web/routers/queue.py index 4a10927..d977f8e 100644 --- a/web/routers/queue.py +++ b/web/routers/queue.py @@ -83,6 +83,7 @@ def prioritize_recent(body: PrioritizeRecent, request: Request) -> dict: n = q.prioritize_recent_hours( request.app.state.db, body.hours ) + q.emit_queue_changed(request.app.state.db, request.app.state.hub) worker = getattr(request.app.state, "sync_worker", None) if worker is not None: worker.kick() @@ -99,6 +100,7 @@ def prioritize(body: Prioritize, request: Request) -> dict: n = q.prioritize( request.app.state.db, body.filenames, body.position ) + q.emit_queue_changed(request.app.state.db, request.app.state.hub) # Kick the worker so a reorder takes effect right away. worker = getattr(request.app.state, "sync_worker", None) if worker is not None: @@ -113,6 +115,7 @@ class Retry(BaseModel): @router.post("/queue/retry", dependencies=[Depends(require_csrf)]) def retry(body: Retry, request: Request) -> dict: n = q.retry(request.app.state.db, body.filenames) + q.emit_queue_changed(request.app.state.db, request.app.state.hub) worker = getattr(request.app.state, "sync_worker", None) if worker is not None: worker.kick() diff --git a/web/routers/settings.py b/web/routers/settings.py index e06b31b..868269f 100644 --- a/web/routers/settings.py +++ b/web/routers/settings.py @@ -53,6 +53,17 @@ def _editable_values(snap) -> dict[str, Any]: "NOMINATIM_EMAIL": snap.nominatim_email, "GEOCODE_ENABLED": snap.geocode_enabled, "DISTANCE_UNITS": snap.distance_units, + "MQTT_ENABLED": snap.mqtt_enabled, + "MQTT_HOST": snap.mqtt_host, + "MQTT_PORT": snap.mqtt_port, + "MQTT_USERNAME": snap.mqtt_username, + "MQTT_PASSWORD": snap.mqtt_password, + "MQTT_TLS": snap.mqtt_tls, + "MQTT_CLIENT_ID": snap.mqtt_client_id, + "MQTT_DISCOVERY_PREFIX": snap.mqtt_discovery_prefix, + "MQTT_NODE_ID": snap.mqtt_node_id, + "MQTT_DISCOVERY_ENABLED": snap.mqtt_discovery_enabled, + "MQTT_QOS": snap.mqtt_qos, } diff --git a/web/services/mqtt.py b/web/services/mqtt.py new file mode 100644 index 0000000..24df56b --- /dev/null +++ b/web/services/mqtt.py @@ -0,0 +1,523 @@ +"""MqttService — connection lifecycle, publish pipeline, command dispatch. + +This module is built up across several tasks. Task 8 contributes the +PublishCoalescer used by MqttService; subsequent tasks add the +connection loop, Hub bridge, periodic refresh, and command handling. +""" +from __future__ import annotations + +import asyncio +import enum +import logging +import os +import ssl +import time +from dataclasses import dataclass +from typing import Any, Awaitable, Callable, Optional + + +Sink = Callable[[str, bytes, bool, int], Awaitable[None]] + + +@dataclass +class _Pending: + payload: bytes + retain: bool + qos: int + min_interval: float + + +class PublishCoalescer: + """Per-topic change-detection + minimum-interval coalescing. + + ``consider`` is called with each candidate publish. If the payload + is unchanged from the last successful publish on this topic, the + call is a no-op. If the topic's minimum interval has not yet + elapsed, the call records the latest payload as pending; the + eventual ``flush_due`` (driven by a tick task) emits one publish + per topic with the most recent value. + """ + + def __init__(self, *, monotonic: Callable[[], float] | None = None) -> None: + self._mono = monotonic or time.monotonic + self._last_payload: dict[str, bytes] = {} + self._last_publish: dict[str, float] = {} + self._pending: dict[str, _Pending] = {} + + async def consider( + self, + topic: str, + payload: bytes, + *, + min_interval: float, + sink: Sink, + retain: bool, + qos: int, + ) -> None: + if self._last_payload.get(topic) == payload: + # Already published this exact payload — also cancel any + # stashed update so a flush_due doesn't re-emit it. + self._pending.pop(topic, None) + return + now = self._mono() + last = self._last_publish.get(topic) + if last is None or (now - last) >= min_interval: + await sink(topic, payload, retain, qos) + self._last_payload[topic] = payload + self._last_publish[topic] = now + self._pending.pop(topic, None) + return + # Still inside the cooldown — stash the latest value. + self._pending[topic] = _Pending( + payload=payload, retain=retain, qos=qos, + min_interval=min_interval, + ) + + async def flush_due(self, sink: Sink) -> None: + """Called periodically (e.g. once a second) to emit any + deadline-elapsed pending publishes.""" + now = self._mono() + for topic, pend in list(self._pending.items()): + last = self._last_publish.get(topic, 0.0) + if (now - last) < pend.min_interval: + continue + await sink(topic, pend.payload, pend.retain, pend.qos) + self._last_payload[topic] = pend.payload + self._last_publish[topic] = now + del self._pending[topic] + + def forget(self, topic: str) -> None: + self._last_payload.pop(topic, None) + self._last_publish.pop(topic, None) + self._pending.pop(topic, None) + + def reset(self) -> None: + self._last_payload.clear() + self._last_publish.clear() + self._pending.clear() + + +log = logging.getLogger("viofosync.mqtt") + + +def entities_affected_by(hub_event_type: str): + """Yield every TOPOLOGY entry that should re-publish in response + to the given Hub event type.""" + from .mqtt_topology import TOPOLOGY + for entity in TOPOLOGY: + if hub_event_type in entity.affected_by_hub_events: + yield entity + + +class ConnState(enum.Enum): + IDLE = "idle" # service not started or settings incomplete + CONNECTING = "connecting" + CONNECTED = "connected" + RECONNECTING = "reconnecting" + ERROR = "error" + DISABLED = "disabled" # MQTT_ENABLED is False + + +class MqttService: + """Manages the MQTT connection lifecycle. + + Built up across several tasks. This task gives it status reporting + and the start/stop methods (which are no-ops until Task 10 adds + the real connection loop).""" + + def __init__(self, *, db, provider, hub, app) -> None: + self._db = db + self._provider = provider + self._hub = hub + self._app = app + self._task: Optional[asyncio.Task] = None + self._stop = asyncio.Event() if hub is not None else None + self._state: ConnState = ConnState.IDLE + self._detail: Optional[str] = None + self._last_published_at: Optional[float] = None + self._coalescer = PublishCoalescer() + + # ---- status ---- + + def _set_state(self, state: ConnState, *, detail: Optional[str] = None) -> None: + self._state = state + self._detail = detail + log.info("mqtt: state=%s detail=%s", state.value, detail) + + def get_status(self) -> dict: + return { + "state": self._state.value, + "detail": self._detail, + "last_published_at": self._last_published_at, + } + + BACKOFF_STEPS = (1.0, 2.0, 5.0, 15.0, 60.0) + + def start(self) -> None: + if self._task is not None and not self._task.done(): + return + self._stop = asyncio.Event() + self._task = asyncio.create_task(self._run(), name="mqtt-service") + + async def stop(self) -> None: + if self._task is None: + return + self._stop.set() + self._task.cancel() + try: + await self._task + except (asyncio.CancelledError, Exception): + pass + self._task = None + self._set_state(ConnState.IDLE, detail=None) + + # ---- main loop ---- + + def _cfg(self) -> dict: + snap = self._provider.get() + client_id = snap.mqtt_client_id + if not client_id: + client_id = f"viofosync-{os.urandom(4).hex()}" + try: + self._provider.update({"MQTT_CLIENT_ID": client_id}, actor="mqtt-service") + except Exception: + log.warning("mqtt: failed to persist generated MQTT_CLIENT_ID") + return { + "host": snap.mqtt_host, + "port": snap.mqtt_port, + "username": snap.mqtt_username or None, + "password": snap.mqtt_password or None, + "tls": snap.mqtt_tls, + "client_id": client_id, + "discovery_prefix": snap.mqtt_discovery_prefix, + "node_id": snap.mqtt_node_id, + "discovery_enabled": snap.mqtt_discovery_enabled, + "qos": snap.mqtt_qos, + "version": self._app.version if self._app is not None else "0.0.0", + "configuration_url": ( + f"http://{snap.host}:{snap.port}/" + if (self._app is not None and snap.host not in ("0.0.0.0", "::", "")) + else "" + ), + } + + async def _run(self) -> None: + import aiomqtt # imported lazily so tests that don't need it don't pay + backoff_idx = 0 + while not self._stop.is_set(): + cfg = self._cfg() + if not cfg["host"]: + self._set_state(ConnState.IDLE, detail="MQTT_HOST not set") + return + try: + self._set_state(ConnState.CONNECTING, + detail=f"{cfg['host']}:{cfg['port']}") + await self._connect_and_loop(aiomqtt, cfg) + backoff_idx = 0 + except asyncio.CancelledError: + raise + except aiomqtt.MqttError as e: + log.warning("mqtt: connection lost (%s); reconnecting", e) + self._set_state(ConnState.RECONNECTING, detail=str(e)) + except Exception as e: + log.exception("mqtt: unexpected error") + self._set_state(ConnState.ERROR, detail=str(e)) + if self._stop.is_set(): + break + delay = self.BACKOFF_STEPS[min(backoff_idx, len(self.BACKOFF_STEPS) - 1)] + try: + await asyncio.wait_for(self._stop.wait(), timeout=delay) + # stop() fired during backoff + return + except asyncio.TimeoutError: + pass + backoff_idx += 1 + + async def _connect_and_loop(self, aiomqtt_mod, cfg: dict) -> None: + will = aiomqtt_mod.Will( + topic=f"{cfg['node_id']}/availability", + payload=b"offline", qos=1, retain=True, + ) + client_kwargs: dict[str, Any] = dict( + hostname=cfg["host"], port=cfg["port"], + username=cfg["username"], password=cfg["password"], + identifier=cfg["client_id"], will=will, keepalive=30, + ) + if cfg["tls"]: + client_kwargs["tls_context"] = ssl.create_default_context() + + async with aiomqtt_mod.Client(**client_kwargs) as client: + self._client = client + await self._on_connected(client, cfg) + self._set_state(ConnState.CONNECTED, + detail=f"{cfg['host']}:{cfg['port']}") + async with asyncio.TaskGroup() as tg: + tg.create_task(self._drain_publishes(client, cfg)) + tg.create_task(self._handle_commands(client, cfg)) + tg.create_task(self._tick(client, cfg)) + + async def _on_connected(self, client, cfg: dict) -> None: + # Subscribe FIRST so any retained command from a previous offline + # window is delivered to us before we announce ourselves available. + from .mqtt_topology import ( + TOPOLOGY, build_command_topic, build_discovery_topic, + build_discovery_payload, + ) + import json as _json + + for entity in TOPOLOGY: + if entity.command_handler is not None: + await client.subscribe( + build_command_topic(entity.object_id, cfg), qos=1, + ) + await client.subscribe( + f"{cfg['node_id']}/cmd/prioritize_recent", qos=1, + ) + + if cfg["discovery_enabled"]: + for entity in TOPOLOGY: + topic = build_discovery_topic(entity.component, + entity.object_id, cfg) + await client.publish( + topic, + _json.dumps(build_discovery_payload(entity, cfg)).encode(), + qos=1, retain=True, + ) + await self._publish_full_state(client, cfg) + + # Announce availability LAST so HA never receives "online" before + # discovery, state, and command subscriptions are in place. + await client.publish( + f"{cfg['node_id']}/availability", b"online", + qos=1, retain=True, + ) + + async def _publish_full_state(self, client, cfg: dict) -> None: + from .mqtt_topology import TOPOLOGY, build_state_topic + snap = self._provider.get() + for entity in TOPOLOGY: + if entity.state_fn is None: + continue + try: + value = entity.state_fn(self._hub, self._db, snap) + except Exception: + log.exception("mqtt: state_fn raised for %s", entity.object_id) + continue + if value is None: + continue + topic = build_state_topic(entity.object_id, cfg) + await self._maybe_publish(client, topic, value.encode(), + retain=True, qos=cfg["qos"], + min_interval=entity.min_publish_interval_s) + + async def _maybe_publish( + self, client, topic: str, payload: bytes, + *, retain: bool, qos: int, min_interval: float, + ) -> None: + async def _sink(t, p, r, q): + await client.publish(t, p, qos=q, retain=r) + self._last_published_at = time.time() + await self._coalescer.consider( + topic, payload, min_interval=min_interval, + sink=_sink, retain=retain, qos=qos, + ) + + async def _drain_publishes(self, client, cfg: dict) -> None: + """Subscribe to the Hub; translate each event into a candidate + publish for every affected entity. Uses an asyncio.Queue to + decouple Hub callbacks from broker I/O.""" + q: asyncio.Queue = asyncio.Queue(maxsize=1024) + + async def _hub_handler(event: dict) -> None: + try: + q.put_nowait(event) + except asyncio.QueueFull: + log.warning("mqtt: hub bridge queue full; dropping event type=%s", + event.get("type")) + + # Hub is the existing Hub() instance; it doesn't expose a + # subscribe API today. The simplest minimal-invasive bridge is + # to wrap its broadcast method so we get a fan-out call before + # the original. Replaceable with a proper subscribe model later. + original_broadcast = self._hub.broadcast + + async def _intercepting_broadcast(event: dict) -> None: + await original_broadcast(event) + await _hub_handler(event) + + self._hub.broadcast = _intercepting_broadcast # type: ignore[assignment] + + # Also intercept schedule_broadcast — that path is used from + # worker threads, and currently runs original broadcast on the + # loop. Need to ensure our fan-out fires there too. + original_schedule = self._hub.schedule_broadcast + + def _intercepting_schedule(running_loop, event: dict) -> None: + try: + asyncio.run_coroutine_threadsafe( + _intercepting_broadcast(event), running_loop, + ) + except RuntimeError: + log.debug("event loop closed, dropping event %s", event) + + self._hub.schedule_broadcast = _intercepting_schedule # type: ignore[assignment] + + try: + from .mqtt_topology import build_state_topic + while not self._stop.is_set(): + event = await q.get() + etype = event.get("type") + if not etype: + continue + snap = self._provider.get() + for entity in entities_affected_by(etype): + if entity.state_fn is None: + continue + try: + value = entity.state_fn(self._hub, self._db, snap) + except Exception: + log.exception("mqtt: state_fn raised for %s", + entity.object_id) + continue + if value is None: + continue + await self._maybe_publish( + client, + build_state_topic(entity.object_id, cfg), + value.encode(), + retain=True, + qos=cfg["qos"], + min_interval=entity.min_publish_interval_s, + ) + finally: + # Restore original broadcast methods so subsequent test + # runs / restarts don't accumulate interceptors. + self._hub.broadcast = original_broadcast # type: ignore[assignment] + self._hub.schedule_broadcast = original_schedule # type: ignore[assignment] + + async def _handle_commands(self, client, cfg: dict) -> None: + from .mqtt_topology import ( + TOPOLOGY, build_command_topic, build_command_handlers, + ) + handlers = build_command_handlers(self._app) + routes: dict[str, Any] = {} + for entity in TOPOLOGY: + if entity.command_handler is None: + continue + routes[build_command_topic(entity.object_id, cfg)] = ( + handlers[entity.object_id] + ) + routes[f"{cfg['node_id']}/cmd/prioritize_recent"] = ( + handlers["prioritize_recent"] + ) + + async for message in client.messages: + topic = message.topic.value + handler = routes.get(topic) + if handler is None: + log.debug("mqtt: unrouted topic %s", topic) + continue + try: + await handler(message.payload) + except Exception: + log.exception("mqtt: handler raised for %s", topic) + + CONNECTION_KEYS = frozenset({ + "MQTT_ENABLED", "MQTT_HOST", "MQTT_PORT", "MQTT_USERNAME", + "MQTT_PASSWORD", "MQTT_TLS", "MQTT_CLIENT_ID", + "MQTT_DISCOVERY_PREFIX", "MQTT_NODE_ID", + }) + + async def on_settings_changed(self, keys: set, snap) -> None: + if not (keys & self.CONNECTION_KEYS): + return + # If node_id or discovery_prefix changed, send delete-payloads + # to every old discovery topic before restarting. + if ( + {"MQTT_NODE_ID", "MQTT_DISCOVERY_PREFIX"} & keys + and getattr(self, "_last_node_id", None) is not None + ): + old_cfg = { + "discovery_prefix": self._last_discovery_prefix, + "node_id": self._last_node_id, + } + from .mqtt_topology import TOPOLOGY, build_discovery_topic + for entity in TOPOLOGY: + topic = build_discovery_topic(entity.component, + entity.object_id, old_cfg) + try: + await self._publish_now(topic, b"", True, 1) + except Exception: + log.exception("mqtt: cleanup publish failed for %s", topic) + + self._last_node_id = getattr(snap, "mqtt_node_id", None) + self._last_discovery_prefix = getattr(snap, "mqtt_discovery_prefix", None) + + await self.stop() + if snap.mqtt_enabled and snap.mqtt_host: + self.start() + + async def _publish_now(self, topic: str, payload: bytes, + retain: bool, qos: int) -> None: + """One-shot publish using a fresh short-lived connection. + Used by node-rename cleanup, which needs to publish to the + *old* topology even after settings have already switched.""" + if self._provider is None: + return + snap = self._provider.get() + if not snap.mqtt_host: + return + import aiomqtt + kwargs = dict( + hostname=snap.mqtt_host, port=snap.mqtt_port, + username=snap.mqtt_username or None, + password=snap.mqtt_password or None, + keepalive=10, + ) + if snap.mqtt_tls: + kwargs["tls_context"] = ssl.create_default_context() + try: + async with aiomqtt.Client(**kwargs) as c: + await c.publish(topic, payload, qos=qos, retain=retain) + except Exception: + log.exception("mqtt: _publish_now failed for %s", topic) + + async def _tick(self, client, cfg: dict) -> None: + from .mqtt_topology import TOPOLOGY, build_state_topic + + async def _sink(t, p, r, q): + await client.publish(t, p, qos=q, retain=r) + self._last_published_at = time.time() + + next_refresh = time.monotonic() + while True: + await asyncio.sleep(1.0) + await self._coalescer.flush_due(_sink) + + now = time.monotonic() + if now < next_refresh: + continue + next_refresh = now + 60.0 + snap = self._provider.get() + for entity in TOPOLOGY: + # Only re-publish poll-sourced entities here. Entities + # with hub-event sources are kept fresh by _drain_publishes. + if entity.state_fn is None: + continue + if entity.object_id not in ("disk_used", "dashcam"): + continue + try: + value = entity.state_fn(self._hub, self._db, snap) + except Exception: + log.exception("mqtt: state_fn raised for %s", + entity.object_id) + continue + if value is None: + continue + await self._maybe_publish( + client, + build_state_topic(entity.object_id, cfg), + value.encode(), + retain=True, qos=cfg["qos"], + min_interval=0.0, + ) diff --git a/web/services/mqtt_state.py b/web/services/mqtt_state.py new file mode 100644 index 0000000..2991310 --- /dev/null +++ b/web/services/mqtt_state.py @@ -0,0 +1,154 @@ +"""Pure state-extraction functions for MQTT entity values. + +Every function has the signature ``(hub, db, snapshot) -> Optional[str]`` +where the string is the exact MQTT payload to publish, or ``None`` to +skip publishing (the entity will appear as Unknown to HA, distinct from +Unavailable which is the LWT-driven state). + +Functions read from ``hub.last_state`` (a dict updated by Hub.broadcast), +the SQLite ``Database``, and the settings ``Snapshot``. No I/O beyond +SQLite, plus whatever the retention service does to compute the +quota-aware used-% for the disk gauge. +""" +from __future__ import annotations + +import datetime as _dt +from typing import Any, Optional + + +def _iso_z(ts: int) -> str: + """ISO 8601 with explicit UTC marker — what HA's timestamp + device_class expects.""" + return ( + _dt.datetime.fromtimestamp(ts, tz=_dt.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%S+00:00") + ) + + +# ---- binary sensors + +def state_dashcam(hub, db, snapshot) -> Optional[str]: + if not snapshot.address: + return "OFF" + val = hub.last_state.get("dashcam_online") + if val is None: + return None + return "ON" if val else "OFF" + + +def state_sync_status(hub, db, snapshot) -> Optional[str]: + sync_state = hub.last_state.get("sync_state") + current_item = hub.last_state.get("current_item") + if not sync_state or not sync_state.get("running"): + return "stopped" + if sync_state.get("paused"): + return "paused" + if current_item: + return "downloading" + # Between items in a sync cycle the in-flight slot is briefly empty + # while the worker writes the GPX sidecar, marks the row done, and + # picks the next clip. Treat "queue still has unfinished work" as + # downloading so HA doesn't flicker to "idle" every few seconds. + with db.conn() as c: + row = c.execute( + "SELECT COUNT(*) AS n FROM download_queue " + "WHERE state IN ('pending', 'downloading')" + ).fetchone() + if row["n"] > 0: + return "downloading" + return "idle" + + +# ---- queue counts + +def _queue_count(db, state: str) -> int: + with db.conn() as c: + row = c.execute( + "SELECT COUNT(*) AS n FROM download_queue WHERE state=?", + (state,), + ).fetchone() + return row["n"] + + +def state_queue_pending(hub, db, snapshot) -> Optional[str]: + return str(_queue_count(db, "pending")) + + +def state_queue_failed(hub, db, snapshot) -> Optional[str]: + return str(_queue_count(db, "failed")) + + +def state_queue_downloading(hub, db, snapshot) -> Optional[str]: + return str(_queue_count(db, "downloading")) + + +# ---- archive + +def state_last_downloaded_clip(hub, db, snapshot) -> Optional[str]: + with db.conn() as c: + row = c.execute( + "SELECT MAX(timestamp) AS m FROM clip_index" + ).fetchone() + ts = row["m"] + if not ts: + return None + return _iso_z(int(ts)) + + +def state_total_clips(hub, db, snapshot) -> Optional[str]: + with db.conn() as c: + row = c.execute("SELECT COUNT(*) AS n FROM clip_index").fetchone() + return str(row["n"]) + + +# ---- current download + +def state_current_filename(hub, db, snapshot) -> Optional[str]: + ci = hub.last_state.get("current_item") + if not ci: + return None + return ci.get("filename") + + +def state_current_progress(hub, db, snapshot) -> Optional[str]: + ci = hub.last_state.get("current_item") or {} + total = ci.get("total") + done = ci.get("bytes") + if not total or done is None: + return None + pct = round(100 * done / total, 1) + return f"{pct}" + + +# ---- disk + +def state_disk_used(hub, db, snapshot) -> Optional[str]: + """Report the higher of (filesystem %, quota %) — that's the + rule closest to triggering retention cleanup. Reusing the + retention service's cache means the sweep and the sensor see + identical numbers. + + Filesystem mode (no quota set): the rule reports the underlying + volume's used %. Quota mode (RECORDINGS_QUOTA_GB > 0): the rule + reports bytes-under-recordings ÷ quota. Independent triggers + (post-cherry-pick) mean both rules can be active at once; we + publish the max so a single HA threshold alerts on either. + """ + from . import retention as _ret + quota = getattr(snapshot, "recordings_quota_gb", 0) or 0 + + # Filesystem rule is always queryable (there's always a mounted + # volume under recordings). Quota rule is opt-in via the setting. + candidates = [] + pct_fs = _ret.disk_used_pct(snapshot.recordings, quota_gb=0) + if pct_fs is not None: + candidates.append(pct_fs) + if quota > 0: + pct_quota = _ret.disk_used_pct(snapshot.recordings, quota_gb=quota) + if pct_quota is not None: + candidates.append(pct_quota) + if not candidates: + return None + return str(int(round(max(candidates)))) + + diff --git a/web/services/mqtt_topology.py b/web/services/mqtt_topology.py new file mode 100644 index 0000000..4f33ad5 --- /dev/null +++ b/web/services/mqtt_topology.py @@ -0,0 +1,384 @@ +"""Entity catalog + discovery payload builders for MQTT. + +The TOPOLOGY list (populated in later tasks) is the single source of +truth for every entity viofosync publishes. Each entry knows: + +* its HA component (sensor/binary_sensor/button), +* its discovery payload shape, +* how to extract its state from the running app (state_fn), +* which Hub event types should trigger a re-publish, +* whether it accepts commands (command_handler). + +Builders in this module are pure functions over EntityDef + a config +dict, so they're trivial to test without a broker. +""" +from __future__ import annotations + +import asyncio +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Optional + +from . import mqtt_state as _st +from . import queue as _q +from . import scanner as _scanner + +log = logging.getLogger("viofosync.mqtt") + + +@dataclass +class EntityDef: + object_id: str + component: str # "sensor" | "binary_sensor" | "button" + name: str + icon: Optional[str] + device_class: Optional[str] + state_class: Optional[str] + unit_of_measurement: Optional[str] + enabled_by_default: bool + min_publish_interval_s: float + state_fn: Optional[Callable] # signature pinned in Task 6 + command_handler: Optional[Callable[[bytes], Awaitable[None]]] + affected_by_hub_events: tuple[str, ...] = field(default_factory=tuple) + + +def build_state_topic(object_id: str, cfg: dict) -> str: + return f"{cfg['node_id']}/{object_id}/state" + + +def build_command_topic(object_id: str, cfg: dict) -> str: + return f"{cfg['node_id']}/{object_id}/cmd" + + +def build_discovery_topic(component: str, object_id: str, cfg: dict) -> str: + return ( + f"{cfg['discovery_prefix']}/{component}/" + f"{cfg['node_id']}/{object_id}/config" + ) + + +def build_availability_topic(cfg: dict) -> str: + return f"{cfg['node_id']}/availability" + + +def build_unique_id(object_id: str, cfg: dict) -> str: + return f"viofosync_{cfg['node_id']}_{object_id}" + + +def _device_manifest(cfg: dict) -> dict: + m = { + "identifiers": [f"viofosync_{cfg['node_id']}"], + "name": "Viofosync", + "model": "viofosync", + "manufacturer": "viofosync", + "sw_version": cfg.get("version", "0.0.0"), + } + # HA rejects the whole discovery message if configuration_url + # is present but not a valid URL — omit when empty. + url = cfg.get("configuration_url", "") + if url: + m["configuration_url"] = url + return m + + +def build_discovery_payload(entity: EntityDef, cfg: dict) -> dict: + """Render the HA discovery `config` payload for one entity.""" + payload: dict = { + "name": entity.name, + "unique_id": build_unique_id(entity.object_id, cfg), + "availability_topic": build_availability_topic(cfg), + "payload_available": "online", + "payload_not_available": "offline", + "enabled_by_default": entity.enabled_by_default, + "device": _device_manifest(cfg), + } + if entity.component in ("sensor", "binary_sensor"): + payload["state_topic"] = build_state_topic(entity.object_id, cfg) + if entity.component == "button": + payload["command_topic"] = build_command_topic(entity.object_id, cfg) + if entity.icon: + payload["icon"] = entity.icon + if entity.device_class: + payload["device_class"] = entity.device_class + if entity.state_class: + payload["state_class"] = entity.state_class + if entity.unit_of_measurement: + payload["unit_of_measurement"] = entity.unit_of_measurement + return payload + + +# Populated in Tasks 4–7. +TOPOLOGY: list[EntityDef] = [ + # --- Binary sensors --- + EntityDef( + object_id="dashcam", + component="binary_sensor", + name="Dashcam", + icon="mdi:cctv", + device_class="connectivity", + state_class=None, + unit_of_measurement=None, + enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=_st.state_dashcam, + command_handler=None, + affected_by_hub_events=("dashcam_online", "dashcam_offline", + "dashcam_reachability_changed"), + ), + EntityDef( + object_id="sync_status", + component="sensor", + name="Sync status", + icon="mdi:sync", + device_class=None, + state_class=None, + unit_of_measurement=None, + enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=_st.state_sync_status, + command_handler=None, + affected_by_hub_events=( + "sync_state", "item_started", "item_finished", "queue_changed", + ), + ), + + # --- Queue --- + EntityDef( + object_id="queue_pending", + component="sensor", + name="Queue pending", + icon="mdi:download", + device_class=None, + state_class="measurement", + unit_of_measurement=None, + enabled_by_default=True, + min_publish_interval_s=1.0, + state_fn=_st.state_queue_pending, + command_handler=None, + affected_by_hub_events=("queue_changed", "item_started", + "item_finished"), + ), + EntityDef( + object_id="queue_failed", + component="sensor", + name="Queue failed", + icon="mdi:alert-circle", + device_class=None, + state_class="measurement", + unit_of_measurement=None, + enabled_by_default=False, + min_publish_interval_s=1.0, + state_fn=_st.state_queue_failed, + command_handler=None, + affected_by_hub_events=("queue_changed",), + ), + EntityDef( + object_id="queue_downloading", + component="sensor", + name="Queue downloading", + icon="mdi:download-circle", + device_class=None, + state_class="measurement", + unit_of_measurement=None, + enabled_by_default=False, + min_publish_interval_s=1.0, + state_fn=_st.state_queue_downloading, + command_handler=None, + affected_by_hub_events=("queue_changed", "item_started", + "item_finished"), + ), + + # --- Archive --- + EntityDef( + object_id="last_downloaded_clip", + component="sensor", + name="Last downloaded clip", + icon="mdi:clock", + device_class="timestamp", + state_class=None, + unit_of_measurement=None, + enabled_by_default=True, + min_publish_interval_s=5.0, + state_fn=_st.state_last_downloaded_clip, + command_handler=None, + affected_by_hub_events=("clip_indexed",), + ), + EntityDef( + object_id="total_clips", + component="sensor", + name="Total clips", + icon="mdi:counter", + device_class=None, + state_class="measurement", + unit_of_measurement=None, + enabled_by_default=False, + min_publish_interval_s=5.0, + state_fn=_st.state_total_clips, + command_handler=None, + affected_by_hub_events=("clip_indexed",), + ), + + # --- Current download --- + EntityDef( + object_id="current_filename", + component="sensor", + name="Current download", + icon="mdi:file-download", + device_class=None, + state_class=None, + unit_of_measurement=None, + enabled_by_default=False, + min_publish_interval_s=2.0, + state_fn=_st.state_current_filename, + command_handler=None, + affected_by_hub_events=("item_started", "item_finished"), + ), + EntityDef( + object_id="current_progress", + component="sensor", + name="Current progress", + icon="mdi:progress-download", + device_class=None, + state_class="measurement", + unit_of_measurement="%", + enabled_by_default=False, + min_publish_interval_s=2.0, + state_fn=_st.state_current_progress, + command_handler=None, + affected_by_hub_events=("item_progress", "item_started", + "item_finished"), + ), + + # --- Disk / sync history --- + EntityDef( + object_id="disk_used", + component="sensor", + name="Disk used", + icon="mdi:harddisk", + device_class=None, + state_class="measurement", + unit_of_measurement="%", + enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=_st.state_disk_used, + command_handler=None, + affected_by_hub_events=(), # poll-only, no hub events + ), +] + + +# Buttons in TOPOLOGY use sentinel `_pending_command` for `command_handler`. +# The real handler is bound at startup via build_command_handlers(app). +async def _pending_command(_payload: bytes) -> None: + raise RuntimeError( + "command handler not bound — call build_command_handlers(app) first" + ) + + +TOPOLOGY.extend([ + EntityDef( + object_id="start_sync", component="button", name="Start sync", + icon="mdi:play", device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, command_handler=_pending_command, + affected_by_hub_events=(), + ), + EntityDef( + object_id="pause_sync", component="button", name="Pause sync", + icon="mdi:pause", device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, command_handler=_pending_command, + affected_by_hub_events=(), + ), + EntityDef( + object_id="skip_current", component="button", name="Skip current download", + icon="mdi:skip-next", device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, command_handler=_pending_command, + affected_by_hub_events=(), + ), + EntityDef( + object_id="refresh_queue", component="button", name="Refresh queue", + icon="mdi:refresh", device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, command_handler=_pending_command, + affected_by_hub_events=(), + ), + EntityDef( + object_id="retry_failed", component="button", name="Retry failed", + icon="mdi:reload-alert", device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, command_handler=_pending_command, + affected_by_hub_events=(), + ), + EntityDef( + object_id="rescan_archive", component="button", name="Rescan archive", + icon="mdi:folder-refresh", device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=None, command_handler=_pending_command, + affected_by_hub_events=(), + ), +]) + + +def build_command_handlers(app: Any) -> dict[str, Any]: + """Return a dict of object_id (and 'prioritize_recent') to async + handlers bound to the running app's services.""" + + async def _start_sync(_p: bytes) -> None: + sw = app.state.sync_worker + sw.resume() + sw.start() + sw.kick() + + async def _pause_sync(_p: bytes) -> None: + app.state.sync_worker.pause() + + async def _skip_current(_p: bytes) -> None: + app.state.sync_worker.skip_current() + + async def _refresh_queue(_p: bytes) -> None: + app.state.sync_worker.kick() + + async def _retry_failed(_p: bytes) -> None: + _q.retry_failed(app.state.db) + _q.emit_queue_changed(app.state.db, app.state.hub) + app.state.sync_worker.kick() + + async def _rescan_archive(_p: bytes) -> None: + snap = app.state.settings_provider.get() + await asyncio.to_thread( + _scanner.scan, app.state.db, snap.recordings, snap.grouping, + app.state.hub, asyncio.get_running_loop(), + ) + + async def _prioritize_recent(payload: bytes) -> None: + try: + body = json.loads(payload) + hours = float(body["hours"]) + except (json.JSONDecodeError, KeyError, TypeError, ValueError): + log.warning("mqtt: prioritize_recent: bad payload %r", payload[:80]) + return + if not (0 < hours <= 168): + log.warning("mqtt: prioritize_recent: hours out of range: %s", hours) + return + _q.prioritize_recent_hours(app.state.db, hours) + _q.emit_queue_changed(app.state.db, app.state.hub) + app.state.sync_worker.kick() + + return { + "start_sync": _start_sync, + "pause_sync": _pause_sync, + "skip_current": _skip_current, + "refresh_queue": _refresh_queue, + "retry_failed": _retry_failed, + "rescan_archive": _rescan_archive, + "prioritize_recent": _prioritize_recent, + } diff --git a/web/services/queue.py b/web/services/queue.py index 820feaa..5dded0a 100644 --- a/web/services/queue.py +++ b/web/services/queue.py @@ -618,3 +618,40 @@ def retry(db: Database, filenames: List[str]) -> int: filenames, ) return cur.rowcount + + +def retry_failed(db: Database) -> int: + """Move every queue item in state ``failed`` back to ``pending``, + resetting attempts. Returns the count updated.""" + with db.write() as c: + cur = c.execute( + "UPDATE download_queue SET state='pending', " + "attempts=0, last_error=NULL WHERE state='failed'" + ) + return cur.rowcount + + +def emit_queue_changed(db: Database, hub, *, loop=None) -> None: + """Broadcast queue_changed; ``loop`` required when calling from + a non-loop thread (sync_worker download thread).""" + if hub is None: + return + with db.conn() as c: + rows = c.execute( + "SELECT state, COUNT(*) AS n FROM download_queue " + "WHERE state IN ('pending','downloading','failed') " + "GROUP BY state" + ).fetchall() + counts = {"pending": 0, "downloading": 0, "failed": 0} + for r in rows: + counts[r["state"]] = r["n"] + event = {"type": "queue_changed", **counts} + import asyncio as _asyncio + try: + running = _asyncio.get_running_loop() + running.create_task(hub.broadcast(event)) + return + except RuntimeError: + pass + if loop is not None: + hub.schedule_broadcast(loop, event) diff --git a/web/services/scanner.py b/web/services/scanner.py index c8b02de..0706b72 100644 --- a/web/services/scanner.py +++ b/web/services/scanner.py @@ -115,7 +115,7 @@ def _iter_clips( ) -def scan(db: Database, destination: str, grouping: str) -> int: +def scan(db: Database, destination: str, grouping: str, hub=None, loop=None) -> int: """Full rescan. Returns the number of rows written. The directory walk runs *without* the DB write lock so that a @@ -124,6 +124,10 @@ def scan(db: Database, destination: str, grouping: str) -> int: single short write transaction. Idempotent: re-running only bumps ``scanned_at``. + + When ``hub`` is provided a ``clip_indexed`` event is broadcast + after the write transaction commits. Pass ``loop`` when calling + from a non-async thread (e.g. via ``asyncio.to_thread``). """ now = int(time.time()) @@ -196,6 +200,15 @@ def scan(db: Database, destination: str, grouping: str) -> int: c.execute("ROLLBACK") raise + if hub is not None: + event = {"type": "clip_indexed", "total": len(seen_paths)} + try: + running = asyncio.get_running_loop() + running.create_task(hub.broadcast(event)) + except RuntimeError: + if loop is not None: + hub.schedule_broadcast(loop, event) + return len(seen_paths) diff --git a/web/services/sync_worker.py b/web/services/sync_worker.py index 7788fd5..001b822 100644 --- a/web/services/sync_worker.py +++ b/web/services/sync_worker.py @@ -475,6 +475,7 @@ async def _refresh_listing_and_reconcile(self) -> bool: "summary": summary, "queue": q.list_all(self.db, limit=200), }) + q.emit_queue_changed(self.db, self.hub) return True async def _cycle(self) -> bool: @@ -540,6 +541,7 @@ async def _cycle(self) -> bool: await asyncio.to_thread( scanner.scan, self.db, snap.recordings, snap.grouping, + self.hub, asyncio.get_running_loop(), ) await scanner.sweep_missing_thumbs( self.db, snap.recordings, @@ -656,6 +658,7 @@ def _blocking(): if ok: q.mark_done(self.db, item.id) + q.emit_queue_changed(self.db, self.hub) return True new_state = q.mark_transient_failure( @@ -664,6 +667,7 @@ def _blocking(): err or "unknown", snap.max_attempts, ) + q.emit_queue_changed(self.db, self.hub) await self.hub.broadcast({ "type": "item_state_change", "filename": item.filename, diff --git a/web/settings.py b/web/settings.py index b04017a..2df8ab8 100644 --- a/web/settings.py +++ b/web/settings.py @@ -71,6 +71,18 @@ class Snapshot: is_unconfigured: bool + mqtt_enabled: bool + mqtt_host: str + mqtt_port: int + mqtt_username: str + mqtt_password: str + mqtt_tls: bool + mqtt_client_id: str + mqtt_discovery_prefix: str + mqtt_node_id: str + mqtt_discovery_enabled: bool + mqtt_qos: int + class SettingsProvider: def __init__( @@ -235,6 +247,17 @@ def _make_snapshot(self, data: dict) -> Snapshot: geocode_enabled=m.GEOCODE_ENABLED, distance_units=m.DISTANCE_UNITS, is_unconfigured=not m.WEB_PASSWORD_HASH, + mqtt_enabled=m.MQTT_ENABLED, + mqtt_host=m.MQTT_HOST, + mqtt_port=m.MQTT_PORT, + mqtt_username=m.MQTT_USERNAME, + mqtt_password=m.MQTT_PASSWORD, + mqtt_tls=m.MQTT_TLS, + mqtt_client_id=m.MQTT_CLIENT_ID, + mqtt_discovery_prefix=m.MQTT_DISCOVERY_PREFIX, + mqtt_node_id=m.MQTT_NODE_ID, + mqtt_discovery_enabled=m.MQTT_DISCOVERY_ENABLED, + mqtt_qos=m.MQTT_QOS, ) # ---------------------------------------------------------- audit log diff --git a/web/settings_schema.py b/web/settings_schema.py index 08ed7ce..d8b304e 100644 --- a/web/settings_schema.py +++ b/web/settings_schema.py @@ -11,7 +11,7 @@ import secrets from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator GROUPING_OPTIONS = ("none", "daily", "weekly", "monthly", "yearly") ENCODER_OPTIONS = ("auto", "software", "videotoolbox", "nvenc", "qsv", "vaapi") @@ -19,6 +19,7 @@ # Valid hostname per RFC 1123 (relaxed) or IPv4/IPv6. _HOSTNAME_RE = re.compile(r"^(?=.{1,253}$)([a-zA-Z0-9][a-zA-Z0-9-]{0,62})(\.[a-zA-Z0-9][a-zA-Z0-9-]{0,62})*$") _IPV4_RE = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$") +_MQTT_NODE_ID_RE = re.compile(r"^[a-z0-9_]{1,32}$") class SettingsModel(BaseModel): @@ -63,6 +64,18 @@ class SettingsModel(BaseModel): GEOCODE_ENABLED: bool = True DISTANCE_UNITS: Literal["km", "miles"] = "km" + MQTT_ENABLED: bool = False + MQTT_HOST: str = "" + MQTT_PORT: int = Field(default=1883, ge=1, le=65535) + MQTT_USERNAME: str = "" + MQTT_PASSWORD: str = "" + MQTT_TLS: bool = False + MQTT_CLIENT_ID: str = "" + MQTT_DISCOVERY_PREFIX: str = "homeassistant" + MQTT_NODE_ID: str = "viofosync" + MQTT_DISCOVERY_ENABLED: bool = True + MQTT_QOS: Literal[0, 1, 2] = 1 + @field_validator("ADDRESS") @classmethod def _validate_address(cls, v: str | None) -> str | None: @@ -82,6 +95,42 @@ def _validate_email(cls, v: str) -> str: raise ValueError("not a valid email address") return v + @field_validator("MQTT_HOST") + @classmethod + def _validate_mqtt_host(cls, v: str) -> str: + if not v: + return "" + v = v.strip() + if _IPV4_RE.match(v) or _HOSTNAME_RE.match(v) or ":" in v: + return v + raise ValueError(f"not a valid MQTT host: {v!r}") + + @field_validator("MQTT_NODE_ID") + @classmethod + def _validate_mqtt_node_id(cls, v: str) -> str: + if not _MQTT_NODE_ID_RE.match(v): + raise ValueError( + "MQTT_NODE_ID must match [a-z0-9_]{1,32}" + ) + return v + + @field_validator("MQTT_DISCOVERY_PREFIX") + @classmethod + def _validate_mqtt_discovery_prefix(cls, v: str) -> str: + if not v: + raise ValueError("MQTT_DISCOVERY_PREFIX must not be empty") + if v.startswith("/") or v.endswith("/"): + raise ValueError( + "MQTT_DISCOVERY_PREFIX must not start/end with '/'" + ) + return v + + @model_validator(mode="after") + def _validate_mqtt_cross_field(self): + if self.MQTT_ENABLED and not self.MQTT_HOST: + raise ValueError("MQTT_HOST is required when MQTT_ENABLED is True") + return self + # Public taxonomy used by the API + UI. EDITABLE_KEYS = { @@ -93,6 +142,10 @@ def _validate_email(cls, v: str) -> str: "RETENTION_PROTECT_RO", "RECORDINGS_QUOTA_GB", "DISTANCE_UNITS", "PIP_POSITION", + "MQTT_ENABLED", "MQTT_HOST", "MQTT_PORT", "MQTT_USERNAME", + "MQTT_PASSWORD", "MQTT_TLS", "MQTT_CLIENT_ID", + "MQTT_DISCOVERY_PREFIX", "MQTT_NODE_ID", + "MQTT_DISCOVERY_ENABLED", "MQTT_QOS", } RESTART_REQUIRED_KEYS = {"WEB_HOST", "WEB_PORT"} READONLY_KEYS = {"PUID", "PGID", "TZ", "RECORDINGS"} diff --git a/web/static/app.js b/web/static/app.js index cfa327e..b72a7db 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -1836,7 +1836,13 @@ function renderSettingsSection(name) { web: renderWebSection, security: renderSecuritySection, system: renderSystemSection, + mqtt: renderMqttSection, }; + // Clear MQTT status polling if navigating away from that section. + if (name !== "mqtt" && _mqttStatusTimer) { + clearInterval(_mqttStatusTimer); + _mqttStatusTimer = null; + } pane.innerHTML = ""; (fns[name] || fns.dashcam)(pane); } @@ -2243,6 +2249,103 @@ function renderSystemSection(pane) { }); } +// ---- MQTT section ---- + +let _mqttStatusTimer = null; + +async function refreshMqttStatus() { + const el = document.getElementById("mqtt-status"); + if (!el) { clearInterval(_mqttStatusTimer); _mqttStatusTimer = null; return; } + try { + const body = await api("/api/mqtt/status"); + const dot = el.querySelector(".dot"); + const text = el.querySelector(".mqtt-status-text"); + dot.className = "dot " + ({ + connected: "green", + connecting: "amber", + reconnecting: "amber", + error: "red", + disabled: "grey", + idle: "grey", + }[body.state] || "grey"); + text.textContent = ({ + connected: `Connected (${body.detail || ""})`, + connecting: `Connecting (${body.detail || ""})`, + reconnecting: `Reconnecting (${body.detail || ""})`, + error: `Error: ${body.detail || ""}`, + disabled: "Disabled", + idle: body.detail || "Not configured", + }[body.state] || body.state); + } catch (_) { /* silently ignore if panel navigated away */ } +} + +function renderMqttSection(pane) { + const hint = document.createElement("p"); + hint.className = "hint"; + hint.textContent = + "Publish state and accept actions over MQTT, with Home Assistant " + + "auto-discovery. See README for the topic structure."; + pane.appendChild(hint); + + renderField(pane, "MQTT_ENABLED", "Enable MQTT", checkbox("MQTT_ENABLED")); + renderField(pane, "MQTT_HOST", "Broker host", textInput("MQTT_HOST")); + renderField(pane, "MQTT_PORT", "Port", + textInput("MQTT_PORT", { type: "number", min: 1, max: 65535 })); + renderField(pane, "MQTT_USERNAME", "Username", textInput("MQTT_USERNAME")); + renderField(pane, "MQTT_PASSWORD", "Password", + textInput("MQTT_PASSWORD", { type: "password" })); + renderField(pane, "MQTT_TLS", "Use TLS", checkbox("MQTT_TLS")); + renderField(pane, "MQTT_CLIENT_ID", "Client ID", textInput("MQTT_CLIENT_ID")); + renderField(pane, "MQTT_NODE_ID", "Node ID", textInput("MQTT_NODE_ID")); + renderField(pane, "MQTT_DISCOVERY_PREFIX", "Discovery prefix", + textInput("MQTT_DISCOVERY_PREFIX")); + renderField(pane, "MQTT_DISCOVERY_ENABLED", "Publish Home Assistant discovery", + checkbox("MQTT_DISCOVERY_ENABLED")); + renderField(pane, "MQTT_QOS", "QoS", select("MQTT_QOS", [0, 1, 2])); + + // Status indicator + const statusEl = document.createElement("p"); + statusEl.id = "mqtt-status"; + statusEl.innerHTML = 'Disabled'; + pane.appendChild(statusEl); + + // Test connection button + const testBtn = document.createElement("button"); + testBtn.type = "button"; + testBtn.id = "mqtt-test-btn"; + testBtn.textContent = "Test connection"; + testBtn.addEventListener("click", async () => { + testBtn.disabled = true; + testBtn.textContent = "Testing…"; + try { + const body = { + host: valueOf("MQTT_HOST"), + port: Number(valueOf("MQTT_PORT") || 1883), + username: valueOf("MQTT_USERNAME"), + password: valueOf("MQTT_PASSWORD"), + tls: !!valueOf("MQTT_TLS"), + client_id: valueOf("MQTT_CLIENT_ID"), + }; + const result = await api("/api/mqtt/test", { + method: "POST", + body: JSON.stringify(body), + }); + alert(result.detail); + } catch (e) { + alert("Test failed: " + (e.message || e)); + } finally { + testBtn.disabled = false; + testBtn.textContent = "Test connection"; + } + }); + pane.appendChild(testBtn); + + // Initial status fetch + start polling (cleared when pane re-renders) + refreshMqttStatus(); + if (_mqttStatusTimer) clearInterval(_mqttStatusTimer); + _mqttStatusTimer = setInterval(refreshMqttStatus, 5000); +} + function showRestartBanner() { if (document.getElementById("banner-restart")) return; const b = document.createElement("div"); diff --git a/web/static/index.html b/web/static/index.html index 0b58abe..a33280c 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -147,6 +147,7 @@

Download manager

Web server Security System + MQTT

Loading…

diff --git a/web/static/styles.css b/web/static/styles.css index 4151e93..4ea784e 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -990,7 +990,7 @@ main { .settings-pane .hint { font-size: 12px; color: var(--muted); - margin: 6px 0 0; + margin: 6px 0 1.5rem; max-width: 60ch; line-height: 1.55; } @@ -1471,6 +1471,27 @@ main { } } +/* ============================================================ + * MQTT settings panel + * ============================================================ */ +#mqtt-panel { margin-top: 1rem; } +#mqtt-panel .mqtt-grid { display: grid; grid-template-columns: 1fr 1fr; gap: .25rem; } +#mqtt-panel .pw-wrap { display: flex; gap: .25rem; } +#mqtt-panel .pw-wrap input { flex: 1; } +#mqtt-status { display: flex; align-items: center; gap: .5rem; margin: .8rem 0; } +#mqtt-status .dot { width: .7rem; height: .7rem; border-radius: 50%; display: inline-block; flex-shrink: 0; } +#mqtt-status .dot.green { background: #19a974; } +#mqtt-status .dot.amber { background: #ffb700; } +#mqtt-status .dot.red { background: #d33; } +#mqtt-status .dot.grey { background: #999; } +.mqtt-actions { display: flex; gap: .5rem; flex-wrap: wrap; margin-top: .5rem; } +.mqtt-actions .primary { + background: var(--accent); + border-color: var(--accent); + color: var(--on-accent); + font-weight: 600; +} + /* ============================================================ * Storage usage card (Settings → Archive Retention) * ============================================================ */ From 71f5bba57428bde69118ded4f7a62d65306446ed Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 28 May 2026 23:09:57 +0100 Subject: [PATCH 04/21] Unified sync_status with four states (downloading/waiting/paused/error) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the previous web UI "Dashcam online/offline" badge and the Home Assistant `sync_status` sensor's `stopped`/`paused`/`downloading`/ `idle` vocabulary with a single four-state status that means the same thing everywhere. Fixes two bugs from the old behaviour: - HA `sync_status` sensor stuck at `downloading` when the dashcam went offline (it only checked sync_state.running, never dashcam_online). - Web UI sync icon kept spinning while the dashcam was offline even though the badge correctly flipped to "Dashcam offline". State model: downloading green spinning arrows active sync in progress waiting orange static arrows dashcam offline or queue empty paused red pause icon user paused error red+ warning icon sticky condition needing action Error conditions (with human-readable reason): - Camera address not configured (ADDRESS unset) - Recordings path not writable - Camera authentication failure (HTTP 401/403) - Filesystem usage above DISK_CRITICAL_PCT (new setting, default 95%) The critical-disk trigger measures *filesystem* percentage, not the retention-aware quota percentage — quota retention is designed to keep the recordings dir at the quota, so the quota metric reads ~100% during normal operation. Architecture: - `web/services/sync_status.py` — pure `compute_sync_status()` is the single source of truth, called by both the MQTT sensor and the Hub. - Hub stores `sync_status` and `sync_status_reason` in `last_state`, broadcasts a `sync_status` event on change so the web UI updates. - Sync worker broadcasts `disk_pct` and stateful `sync_error` events that feed the compute function. - MQTT `sync_status` sensor exposes a `reason` JSON attribute; new `attrs_fn` infrastructure on `EntityDef` adds `json_attributes_topic` to discovery payloads. - Web UI badge shows "Error: " inline so users see the cause without having to hover. Breaking change: HA `sensor.viofosync_sync_status` published values change from {stopped, paused, downloading, idle} to {downloading, waiting, paused, error}. Automations matching the old strings need updating — see CHANGELOG. --- CHANGELOG.md | 21 +++ tests/test_hub.py | 212 ++++++++++++++++++++++++ tests/test_mqtt_discovery_payload.py | 41 +++++ tests/test_mqtt_state_extraction.py | 67 ++++---- tests/test_mqtt_topology.py | 21 +++ tests/test_settings_schema.py | 39 +++++ tests/test_startup_sync_status.py | 21 +++ tests/test_sync_status.py | 181 ++++++++++++++++++++ tests/test_sync_worker_disk_pct.py | 89 ++++++++++ tests/test_sync_worker_error_signals.py | 97 +++++++++++ web/app.py | 11 +- web/services/hub.py | 68 +++++++- web/services/mqtt.py | 30 ++++ web/services/mqtt_state.py | 33 ++-- web/services/mqtt_topology.py | 13 +- web/services/retention.py | 17 ++ web/services/sync_status.py | 79 +++++++++ web/services/sync_worker.py | 78 ++++++++- web/settings.py | 2 + web/settings_schema.py | 16 +- web/static/app.js | 106 +++++++----- web/static/index.html | 6 +- web/static/styles.css | 37 ++++- 23 files changed, 1182 insertions(+), 103 deletions(-) create mode 100644 tests/test_startup_sync_status.py create mode 100644 tests/test_sync_status.py create mode 100644 tests/test_sync_worker_disk_pct.py create mode 100644 tests/test_sync_worker_error_signals.py create mode 100644 web/services/sync_status.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b2d85f9..fd66d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,27 @@ action buttons; idle traffic is zero thanks to per-topic change detection and coalescing. LWT keeps HA's view consistent with viofosync's actual state. +- `sync_status` now reports `error` for sticky problems: missing + `ADDRESS` configuration, recordings path not writable, camera + authentication failure (HTTP 401/403), or disk usage at/above + `DISK_CRITICAL_PCT` (new setting, default 95%). +- The HA `sync_status` sensor exposes a `reason` JSON attribute + populated when state is `error`. Surface it in Lovelace with + `state_attr('sensor.viofosync_sync_status', 'reason')`. +- New `DISK_CRITICAL_PCT` setting (Snapshot field `disk_critical_pct`) + configures the disk-pressure threshold above which sync goes into + `error`. Must be `>= RETENTION_DISK_PCT`. + +### Changed +- Unified `sync_status` to four states: `downloading`, `waiting`, + `paused`, `error`. Replaces the previous `stopped` / `paused` / + `downloading` / `idle` vocabulary on the Home Assistant + `sensor.viofosync_sync_status` entity, and replaces the separate + "Dashcam online / offline" badge in the web UI with a single status + badge. If you have HA automations matching the previous strings, + update them: `idle` and `stopped` map to `waiting` (or `paused` + when sync is fully stopped). Connection state is still reflected + via the existing `binary_sensor.viofosync_dashcam`. ## v2.1 — 2026-05-16 diff --git a/tests/test_hub.py b/tests/test_hub.py index d0a6e0a..c3032a1 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -67,3 +67,215 @@ async def test_connect_swallows_oserror_during_snapshot() -> None: ws = _FakeWS(send_raises=OSError("broken pipe")) await hub.connect(ws) assert ws not in hub._clients + + +# ---- last_state accumulation for new event types ---- + +async def test_broadcast_sync_error_stored_in_last_state() -> None: + hub = Hub() + await hub.broadcast({ + "type": "sync_error", + "kind": "recordings_unwritable", + "message": "recordings path not writable", + }) + assert hub.last_state["sync_error"] == { + "kind": "recordings_unwritable", + "message": "recordings path not writable", + } + + +async def test_broadcast_sync_error_with_kind_none_clears() -> None: + """The worker sends sync_error with kind=None to clear a previously + sticky error.""" + hub = Hub() + await hub.broadcast({ + "type": "sync_error", "kind": "config", "message": "x", + }) + assert hub.last_state["sync_error"] is not None + await hub.broadcast({"type": "sync_error", "kind": None, "message": None}) + assert hub.last_state["sync_error"] is None + + +async def test_broadcast_disk_pct_stored_in_last_state() -> None: + hub = Hub() + await hub.broadcast({"type": "disk_pct", "pct": 87.3}) + assert hub.last_state["disk_pct"] == 87.3 + + +async def test_initial_last_state_includes_new_keys() -> None: + """Newly-constructed Hub exposes the new keys as None so consumers + that read them at startup don't KeyError.""" + hub = Hub() + assert "sync_error" in hub.last_state + assert "disk_pct" in hub.last_state + assert "sync_status" in hub.last_state + assert "sync_status_reason" in hub.last_state + assert hub.last_state["sync_error"] is None + assert hub.last_state["disk_pct"] is None + assert hub.last_state["sync_status"] is None + assert hub.last_state["sync_status_reason"] is None + + +import types as _types + + +def _stub_provider(**snap_overrides): + """Tiny provider stub with .get() returning a settings-like object.""" + base = dict(address="192.168.1.50", recordings="/r", disk_critical_pct=95) + base.update(snap_overrides) + snap = _types.SimpleNamespace(**base) + return _types.SimpleNamespace(get=lambda: snap) + + +class _RecordingWS: + def __init__(self): + self.sent = [] + async def accept(self): pass + async def send_json(self, payload): self.sent.append(payload) + + +async def test_hub_emits_sync_status_after_dashcam_offline() -> None: + """Driving the dashcam offline mid-download should produce a follow-up + sync_status event with state="waiting".""" + hub = Hub(settings_provider=_stub_provider()) + ws = _RecordingWS() + await hub.connect(ws) + ws.sent.clear() + # Worker says: running, dashcam was online, mid-item. + await hub.broadcast({"type": "sync_state", "running": True, "paused": False}) + await hub.broadcast({"type": "dashcam_online"}) + await hub.broadcast({"type": "item_started", "filename": "x.mp4", "total": 100}) + # At this point the status should be "downloading". + assert hub.last_state["sync_status"] == "downloading" + ws.sent.clear() + # Dashcam drives away. + await hub.broadcast({"type": "dashcam_offline"}) + assert hub.last_state["sync_status"] == "waiting" + # A sync_status follow-up event was emitted. + follow_ups = [e for e in ws.sent if e.get("type") == "sync_status"] + assert follow_ups == [{"type": "sync_status", + "status": "waiting", "reason": None}] + + +async def test_hub_does_not_emit_sync_status_when_unchanged() -> None: + """Re-broadcasting the same upstream event (or one that doesn't + flip status) must not produce duplicate sync_status events.""" + hub = Hub(settings_provider=_stub_provider()) + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "sync_state", "running": True, "paused": False}) + await hub.broadcast({"type": "dashcam_online"}) + ws.sent.clear() + # Second dashcam_online — status is still "waiting" (no current item), + # same as before, so no follow-up event. + await hub.broadcast({"type": "dashcam_online"}) + follow_ups = [e for e in ws.sent if e.get("type") == "sync_status"] + assert follow_ups == [] + + +async def test_hub_emits_sync_status_error_with_reason() -> None: + hub = Hub(settings_provider=_stub_provider()) + ws = _RecordingWS() + await hub.connect(ws) + ws.sent.clear() + await hub.broadcast({ + "type": "sync_error", + "kind": "recordings_unwritable", + "message": "recordings path not writable", + }) + assert hub.last_state["sync_status"] == "error" + follow_ups = [e for e in ws.sent if e.get("type") == "sync_status"] + assert follow_ups == [{ + "type": "sync_status", + "status": "error", + "reason": "recordings path not writable", + }] + + +async def test_hub_emits_sync_status_when_disk_crosses_threshold() -> None: + hub = Hub(settings_provider=_stub_provider(disk_critical_pct=95)) + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "sync_state", "running": True, "paused": False}) + await hub.broadcast({"type": "dashcam_online"}) + ws.sent.clear() + await hub.broadcast({"type": "disk_pct", "pct": 80.0}) + assert hub.last_state["sync_status"] == "waiting" # still under threshold + follow_ups_before = [e for e in ws.sent if e.get("type") == "sync_status"] + assert follow_ups_before == [] + ws.sent.clear() + await hub.broadcast({"type": "disk_pct", "pct": 96.4}) + assert hub.last_state["sync_status"] == "error" + follow_ups_after = [e for e in ws.sent if e.get("type") == "sync_status"] + assert follow_ups_after == [{ + "type": "sync_status", "status": "error", "reason": "disk 96% full", + }] + + +async def test_hub_initial_snapshot_carries_sync_status() -> None: + hub = Hub(settings_provider=_stub_provider()) + # Drive into a known state before any client connects. + await hub.broadcast({"type": "sync_state", "running": True, "paused": False}) + await hub.broadcast({"type": "dashcam_offline"}) + ws = _RecordingWS() + await hub.connect(ws) + # First message after connect is the snapshot. + assert ws.sent[0]["type"] == "snapshot" + assert ws.sent[0]["state"]["sync_status"] == "waiting" + + +async def test_hub_snapshot_carries_sync_status_reason() -> None: + """The WS snapshot must include both sync_status and the reason so + a freshly-connected client (e.g. after a page refresh) renders the + error reason in the badge without waiting for a state change.""" + hub = Hub(settings_provider=_stub_provider()) + await hub.broadcast({"type": "sync_state", "running": True, "paused": False}) + await hub.broadcast({ + "type": "sync_error", "kind": "recordings_unwritable", + "message": "recordings path not writable", + }) + ws = _RecordingWS() + await hub.connect(ws) + snap = ws.sent[0] + assert snap["type"] == "snapshot" + assert snap["state"]["sync_status"] == "error" + assert snap["state"]["sync_status_reason"] == "recordings path not writable" + + +async def test_hub_rebroadcasts_when_reason_changes_but_status_does_not() -> None: + """When status stays 'error' but the reason text changes (e.g. + disk pct climbs from 95% to 99%), the hub MUST re-emit so the UI + badge updates.""" + hub = Hub(settings_provider=_stub_provider(disk_critical_pct=95)) + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "sync_state", "running": True, "paused": False}) + await hub.broadcast({"type": "disk_pct", "pct": 95.0}) + assert hub.last_state["sync_status"] == "error" + assert hub.last_state["sync_status_reason"] == "disk 95% full" + ws.sent.clear() + # Disk climbs — same status, new reason. + await hub.broadcast({"type": "disk_pct", "pct": 99.0}) + follow_ups = [e for e in ws.sent if e.get("type") == "sync_status"] + assert follow_ups == [{ + "type": "sync_status", "status": "error", "reason": "disk 99% full", + }] + assert hub.last_state["sync_status_reason"] == "disk 99% full" + + +async def test_hub_compute_exception_does_not_break_broadcast() -> None: + """If status computation raises, the upstream event still propagates + and no sync_status event is emitted.""" + # Use a provider whose .get() raises, to force compute to fail. + class _Boom: + def get(self_): + raise RuntimeError("boom") + hub = Hub(settings_provider=_Boom()) + ws = _RecordingWS() + await hub.connect(ws) + ws.sent.clear() + await hub.broadcast({"type": "dashcam_online"}) + # Upstream event still delivered. + assert any(e.get("type") == "dashcam_online" for e in ws.sent) + # No sync_status follow-up. + assert not any(e.get("type") == "sync_status" for e in ws.sent) diff --git a/tests/test_mqtt_discovery_payload.py b/tests/test_mqtt_discovery_payload.py index 90423bf..9fa6df6 100644 --- a/tests/test_mqtt_discovery_payload.py +++ b/tests/test_mqtt_discovery_payload.py @@ -170,3 +170,44 @@ def test_discovery_payload_disabled_by_default(): ) payload = build_discovery_payload(entity, _cfg()) assert payload["enabled_by_default"] is False + + +def test_discovery_payload_includes_json_attributes_topic_when_attrs_fn(): + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, build_attrs_topic, + ) + + def _stub_state(hub, db, snap): return "x" + def _stub_attrs(hub, db, snap): return {"reason": None} + + ent = EntityDef( + object_id="demo", component="sensor", name="Demo", + icon=None, device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=_stub_state, command_handler=None, + attrs_fn=_stub_attrs, + ) + cfg = {"node_id": "vfs", "discovery_prefix": "homeassistant", + "version": "0.0.0", "configuration_url": ""} + payload = build_discovery_payload(ent, cfg) + assert payload["json_attributes_topic"] == build_attrs_topic("demo", cfg) + + +def test_discovery_payload_omits_json_attributes_topic_without_attrs_fn(): + from web.services.mqtt_topology import ( + EntityDef, build_discovery_payload, + ) + + def _stub_state(hub, db, snap): return "x" + ent = EntityDef( + object_id="demo2", component="sensor", name="Demo2", + icon=None, device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, + state_fn=_stub_state, command_handler=None, + ) + cfg = {"node_id": "vfs", "discovery_prefix": "homeassistant", + "version": "0.0.0", "configuration_url": ""} + payload = build_discovery_payload(ent, cfg) + assert "json_attributes_topic" not in payload diff --git a/tests/test_mqtt_state_extraction.py b/tests/test_mqtt_state_extraction.py index 1052f76..28112d9 100644 --- a/tests/test_mqtt_state_extraction.py +++ b/tests/test_mqtt_state_extraction.py @@ -16,6 +16,7 @@ def _stub_snapshot(**kwargs): recordings=".", enable_scheduled_sync=True, retention_max_days=0, + disk_critical_pct=95, ) base.update(kwargs) return types.SimpleNamespace(**base) @@ -76,82 +77,80 @@ def test_state_dashcam_unknown_when_no_address(): None, _stub_snapshot(address="")) == "OFF" -def test_state_sync_status_stopped_when_missing(): +def test_state_sync_status_paused_when_no_sync_state(): from web.services.mqtt_state import state_sync_status - # sync_state key absent → stopped hub = _hub_with_state({}) - assert state_sync_status(hub, None, _stub_snapshot()) == "stopped" + assert state_sync_status(hub, None, _stub_snapshot()) == "paused" -def test_state_sync_status_stopped_when_not_running(): +def test_state_sync_status_paused_when_not_running(): from web.services.mqtt_state import state_sync_status hub = _hub_with_state({"sync_state": {"running": False, "paused": False}}) - assert state_sync_status(hub, None, _stub_snapshot()) == "stopped" + assert state_sync_status(hub, None, _stub_snapshot()) == "paused" -def test_state_sync_status_paused(): +def test_state_sync_status_paused_when_paused_flag(): from web.services.mqtt_state import state_sync_status hub = _hub_with_state({"sync_state": {"running": True, "paused": True}}) assert state_sync_status(hub, None, _stub_snapshot()) == "paused" -def test_state_sync_status_downloading(): +def test_state_sync_status_downloading_when_current_item(): from web.services.mqtt_state import state_sync_status hub = _hub_with_state({ "sync_state": {"running": True, "paused": False}, - "current_item": {"filename": "x.MP4"}, + "dashcam_online": True, + "current_item": {"filename": "x.mp4"}, }) assert state_sync_status(hub, None, _stub_snapshot()) == "downloading" -def test_state_sync_status_idle_when_queue_empty(tmp_path): - """No current item AND no pending/downloading rows → idle.""" +def test_state_sync_status_waiting_when_no_current_item(): from web.services.mqtt_state import state_sync_status - db = _db_with_queue(tmp_path, []) hub = _hub_with_state({ "sync_state": {"running": True, "paused": False}, + "dashcam_online": True, "current_item": None, }) - assert state_sync_status(hub, db, _stub_snapshot()) == "idle" + assert state_sync_status(hub, None, _stub_snapshot()) == "waiting" -def test_state_sync_status_downloading_between_items(tmp_path): - """No current item but queue still has pending rows → downloading - (covers the gap between consecutive clips in a sync cycle).""" +def test_state_sync_status_waiting_when_dashcam_offline(): from web.services.mqtt_state import state_sync_status - db = _db_with_queue(tmp_path, [("next.MP4", "pending")]) hub = _hub_with_state({ "sync_state": {"running": True, "paused": False}, - "current_item": None, + "dashcam_online": False, }) - assert state_sync_status(hub, db, _stub_snapshot()) == "downloading" + assert state_sync_status(hub, None, _stub_snapshot()) == "waiting" -def test_state_sync_status_downloading_while_row_marked_downloading(tmp_path): - """A row in state='downloading' also counts as work in progress - even if hub.last_state hasn't seen item_started yet.""" +def test_state_sync_status_error_when_address_unset(): from web.services.mqtt_state import state_sync_status - db = _db_with_queue(tmp_path, [("x.MP4", "downloading")]) hub = _hub_with_state({ "sync_state": {"running": True, "paused": False}, - "current_item": None, + "dashcam_online": True, }) - assert state_sync_status(hub, db, _stub_snapshot()) == "downloading" + assert state_sync_status( + hub, None, _stub_snapshot(address=None), + ) == "error" -def test_state_sync_status_idle_ignores_done_and_failed(tmp_path): - """Failed/done rows don't drag status back to downloading.""" - from web.services.mqtt_state import state_sync_status - db = _db_with_queue(tmp_path, [ - ("done.MP4", "done"), - ("fail.MP4", "failed"), - ("gone.MP4", "gone"), - ]) +def test_attrs_sync_status_carries_reason_when_error(): + from web.services.mqtt_state import attrs_sync_status + hub = _hub_with_state({}) + attrs = attrs_sync_status(hub, None, _stub_snapshot(address=None)) + assert attrs == {"reason": "camera address not configured"} + + +def test_attrs_sync_status_reason_none_when_not_error(): + from web.services.mqtt_state import attrs_sync_status hub = _hub_with_state({ "sync_state": {"running": True, "paused": False}, - "current_item": None, + "dashcam_online": True, + "current_item": {"filename": "x.mp4"}, }) - assert state_sync_status(hub, db, _stub_snapshot()) == "idle" + attrs = attrs_sync_status(hub, None, _stub_snapshot()) + assert attrs == {"reason": None} # ---- queue counts diff --git a/tests/test_mqtt_topology.py b/tests/test_mqtt_topology.py index 29a0549..0dbb306 100644 --- a/tests/test_mqtt_topology.py +++ b/tests/test_mqtt_topology.py @@ -93,3 +93,24 @@ def test_command_handler_present_only_on_buttons(): for e in TOPOLOGY: if e.command_handler is not None: assert e.component == "button", e.object_id + + +def test_sync_status_entity_lists_new_affected_events(): + from web.services.mqtt_topology import TOPOLOGY + entity = next(e for e in TOPOLOGY if e.object_id == "sync_status") + events = set(entity.affected_by_hub_events) + assert "dashcam_online" in events + assert "dashcam_offline" in events + assert "disk_pct" in events + assert "sync_error" in events + # Plus the original ones + assert "sync_state" in events + assert "item_started" in events + assert "item_finished" in events + + +def test_sync_status_entity_has_attrs_fn(): + from web.services.mqtt_topology import TOPOLOGY + from web.services.mqtt_state import attrs_sync_status + entity = next(e for e in TOPOLOGY if e.object_id == "sync_status") + assert entity.attrs_fn is attrs_sync_status diff --git a/tests/test_settings_schema.py b/tests/test_settings_schema.py index fb7e666..d6e483d 100644 --- a/tests/test_settings_schema.py +++ b/tests/test_settings_schema.py @@ -170,3 +170,42 @@ def test_pip_position_default_is_top_right() -> None: SettingsModel(PIP_POSITION=pos) with pytest.raises(ValidationError): SettingsModel(PIP_POSITION="middle") + + +def test_disk_critical_pct_default_is_95(): + from web.settings_schema import SettingsModel + m = SettingsModel() + assert m.DISK_CRITICAL_PCT == 95 + + +def test_disk_critical_pct_validates_range(): + from web.settings_schema import SettingsModel + import pytest as _pt + with _pt.raises(Exception): + SettingsModel(DISK_CRITICAL_PCT=101) + with _pt.raises(Exception): + SettingsModel(DISK_CRITICAL_PCT=-1) + # Boundaries OK + assert SettingsModel(DISK_CRITICAL_PCT=0).DISK_CRITICAL_PCT == 0 + assert SettingsModel(DISK_CRITICAL_PCT=100).DISK_CRITICAL_PCT == 100 + + +def test_disk_critical_pct_must_exceed_retention_threshold(): + """A critical threshold below the retention threshold would fire before + retention even tried — reject the combination at validation time.""" + from web.settings_schema import SettingsModel + import pytest as _pt + with _pt.raises(Exception): + SettingsModel(RETENTION_DISK_PCT=90, DISK_CRITICAL_PCT=80) + # Equal is allowed (no overlap means no firing before retention) + SettingsModel(RETENTION_DISK_PCT=90, DISK_CRITICAL_PCT=95) + + +def test_snapshot_exposes_disk_critical_pct(): + from web.settings import SettingsProvider + import tempfile + with tempfile.TemporaryDirectory() as d: + sp = SettingsProvider(config_path=f"{d}/c.json", + env_file_path=f"{d}/v.env", + recordings_dir=d) + assert sp.get().disk_critical_pct == 95 diff --git a/tests/test_startup_sync_status.py b/tests/test_startup_sync_status.py new file mode 100644 index 0000000..2560cff --- /dev/null +++ b/tests/test_startup_sync_status.py @@ -0,0 +1,21 @@ +"""When ADDRESS is unset at startup, the Hub's sync_status must be +'error' before any client connects — otherwise the very first MQTT +publish or WS snapshot would show 'paused'.""" +from __future__ import annotations + +import types + +from web.services.hub import Hub +from web.services.sync_status import compute_sync_status + + +def test_hub_sync_status_is_error_when_address_unset(): + snap = types.SimpleNamespace( + address=None, recordings="/r", disk_critical_pct=95, + ) + provider = types.SimpleNamespace(get=lambda: snap) + hub = Hub(settings_provider=provider) + # No events yet, but compute is total — should report error. + state, reason = compute_sync_status(hub, None, snap) + assert state == "error" + assert reason == "camera address not configured" diff --git a/tests/test_sync_status.py b/tests/test_sync_status.py new file mode 100644 index 0000000..54a9223 --- /dev/null +++ b/tests/test_sync_status.py @@ -0,0 +1,181 @@ +"""Tests for the pure compute_sync_status() function. + +Inputs come from hub.last_state and the settings snapshot. Output is +(state, reason) — never raises. Precedence: error > paused > downloading > waiting. +""" +from __future__ import annotations + +import types + +import pytest + +from web.services.sync_status import compute_sync_status + + +def _hub(**state): + return types.SimpleNamespace(last_state=state) + + +def _snap(**kwargs): + base = dict( + address="192.168.1.50", + recordings="/recordings", + disk_critical_pct=95, + ) + base.update(kwargs) + return types.SimpleNamespace(**base) + + +# ---- error precedence (highest) ---- + +def test_error_when_address_unset(): + hub = _hub(sync_state={"running": True, "paused": False}, + dashcam_online=True) + snap = _snap(address=None) + state, reason = compute_sync_status(hub, None, snap) + assert state == "error" + assert reason == "camera address not configured" + + +def test_error_when_address_empty_string(): + hub = _hub(sync_state={"running": True, "paused": False}, + dashcam_online=True) + snap = _snap(address="") + state, reason = compute_sync_status(hub, None, snap) + assert state == "error" + + +def test_error_when_sync_error_set_in_state(): + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=True, + sync_error={"kind": "recordings_unwritable", + "message": "recordings path not writable"}, + ) + state, reason = compute_sync_status(hub, None, _snap()) + assert state == "error" + assert reason == "recordings path not writable" + + +def test_error_when_disk_pct_at_or_above_critical(): + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=True, + disk_pct=95.0, + ) + state, reason = compute_sync_status(hub, None, _snap(disk_critical_pct=95)) + assert state == "error" + assert reason == "disk 95% full" + + +def test_error_disk_message_rounds_to_integer(): + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=True, + disk_pct=96.7, + ) + state, reason = compute_sync_status(hub, None, _snap(disk_critical_pct=95)) + assert reason == "disk 97% full" + + +def test_no_error_when_disk_critical_disabled(): + """disk_critical_pct=0 disables the check, even at 100%.""" + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=True, + disk_pct=100.0, + ) + state, _ = compute_sync_status(hub, None, _snap(disk_critical_pct=0)) + assert state != "error" + + +def test_error_precedence_over_paused(): + hub = _hub( + sync_state={"running": True, "paused": True}, + dashcam_online=True, + sync_error={"kind": "config", "message": "camera address not configured"}, + ) + state, _ = compute_sync_status(hub, None, _snap(address=None)) + assert state == "error" + + +# ---- paused (worker not running counts as paused) ---- + +def test_paused_when_running_and_paused_flag_set(): + hub = _hub(sync_state={"running": True, "paused": True}, + dashcam_online=True) + state, reason = compute_sync_status(hub, None, _snap()) + assert state == "paused" + assert reason is None + + +def test_paused_when_worker_not_running(): + hub = _hub(sync_state={"running": False, "paused": False}, + dashcam_online=True) + state, _ = compute_sync_status(hub, None, _snap()) + assert state == "paused" + + +def test_paused_when_sync_state_missing_entirely(): + """No sync_state yet — treat as paused (worker hasn't reported in).""" + hub = _hub(dashcam_online=True) + state, _ = compute_sync_status(hub, None, _snap()) + assert state == "paused" + + +# ---- downloading ---- + +def test_downloading_when_current_item_present(): + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=True, + current_item={"filename": "x.mp4", "total": 100, "bytes": 50}, + ) + state, reason = compute_sync_status(hub, None, _snap()) + assert state == "downloading" + assert reason is None + + +# ---- waiting (default for running-but-not-downloading) ---- + +def test_waiting_when_dashcam_offline(): + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=False, + current_item={"filename": "x.mp4"}, # stale — dashcam is gone + ) + state, reason = compute_sync_status(hub, None, _snap()) + assert state == "waiting" + assert reason is None + + +def test_waiting_when_running_no_current_item(): + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=True, + current_item=None, + ) + state, _ = compute_sync_status(hub, None, _snap()) + assert state == "waiting" + + +def test_waiting_when_dashcam_online_unknown(): + """First moments after startup: dashcam_online is None. Don't flicker + to downloading. Treat as waiting.""" + hub = _hub( + sync_state={"running": True, "paused": False}, + dashcam_online=None, + ) + state, _ = compute_sync_status(hub, None, _snap()) + assert state == "waiting" + + +# ---- robustness ---- + +def test_never_raises_on_garbage_state(): + """Any unexpected shape in last_state must resolve to waiting, + never raise — the UI must never blank out.""" + hub = _hub(sync_state="not a dict", + current_item=42, dashcam_online="yes") + state, _ = compute_sync_status(hub, None, _snap()) + assert state in ("downloading", "waiting", "paused", "error") diff --git a/tests/test_sync_worker_disk_pct.py b/tests/test_sync_worker_disk_pct.py new file mode 100644 index 0000000..864e46f --- /dev/null +++ b/tests/test_sync_worker_disk_pct.py @@ -0,0 +1,89 @@ +"""The sync worker broadcasts disk_pct so compute_sync_status can flip +to error when the disk is full. + +The metric broadcast here is *filesystem* percentage — the OS-level +"how full is the disk we're writing to". It explicitly does NOT use +the quota-aware ``disk_used_pct`` because quota retention is designed +to keep the recordings dir *at* the quota, so the quota-aware metric +would read ~100% during normal operation and trigger a spurious +critical-disk error every cycle. +""" +from __future__ import annotations + +import types + +from web.services.sync_worker import SyncWorker + + +class _Hub: + def __init__(self): + self.events = [] + async def broadcast(self, event): + self.events.append(event) + + +def _make_worker(snap, hub): + sw = SyncWorker.__new__(SyncWorker) + sw.hub = hub + sw._provider = types.SimpleNamespace(get=lambda: snap) + sw._loop = None + return sw + + +async def test_emit_disk_pct_uses_filesystem_pct_not_quota(monkeypatch): + """When a quota is configured, _emit_disk_pct MUST broadcast the + filesystem percentage, not the quota percentage. Quota retention + keeps the dir at quota by design — using quota % here would trip + the critical-disk error perpetually.""" + snap = types.SimpleNamespace( + recordings="/r", + recordings_quota_gb=3100, # quota IS set + ) + hub = _Hub() + sw = _make_worker(snap, hub) + + # Quota-aware reading would be ~97% (this should NOT be what we broadcast) + monkeypatch.setattr( + "web.services.retention.disk_used_pct", + lambda recordings, quota_gb=0: 97.0, + ) + # Filesystem reading: 87% (this is what we SHOULD broadcast) + monkeypatch.setattr( + "web.services.retention.filesystem_used_pct", + lambda recordings: 87.0, + ) + + await sw._emit_disk_pct() + assert hub.events == [{"type": "disk_pct", "pct": 87.0}] + + +async def test_emit_disk_pct_uses_filesystem_pct_when_no_quota(monkeypatch): + """Without a quota, the filesystem percentage is still what we want. + Same code path — the broadcast doesn't change behaviour based on quota.""" + snap = types.SimpleNamespace( + recordings="/r", + recordings_quota_gb=0, + ) + hub = _Hub() + sw = _make_worker(snap, hub) + monkeypatch.setattr( + "web.services.retention.filesystem_used_pct", + lambda recordings: 42.5, + ) + + await sw._emit_disk_pct() + assert hub.events == [{"type": "disk_pct", "pct": 42.5}] + + +async def test_emit_disk_pct_swallows_none(monkeypatch): + """filesystem_used_pct returns None when the path is missing — + don't emit a bogus event.""" + snap = types.SimpleNamespace(recordings="/missing", recordings_quota_gb=0) + hub = _Hub() + sw = _make_worker(snap, hub) + monkeypatch.setattr( + "web.services.retention.filesystem_used_pct", + lambda recordings: None, + ) + await sw._emit_disk_pct() + assert hub.events == [] diff --git a/tests/test_sync_worker_error_signals.py b/tests/test_sync_worker_error_signals.py new file mode 100644 index 0000000..50c8007 --- /dev/null +++ b/tests/test_sync_worker_error_signals.py @@ -0,0 +1,97 @@ +"""Sync worker emits stateful sync_error events for conditions that +prevent normal operation: recordings path unwritable, auth failures.""" +from __future__ import annotations + +import os +import types + +from web.services.sync_worker import SyncWorker + + +def _make_worker(snap, hub): + sw = SyncWorker.__new__(SyncWorker) + sw.hub = hub + sw._provider = types.SimpleNamespace(get=lambda: snap) + sw._loop = None + sw._last_error_kind = None + return sw + + +class _RecordingHub: + def __init__(self): + self.events = [] + async def broadcast(self, event): + self.events.append(event) + + +async def test_check_recordings_unwritable_emits_error(tmp_path): + bad = tmp_path / "nope" # doesn't exist + snap = types.SimpleNamespace(recordings=str(bad)) + hub = _RecordingHub() + sw = _make_worker(snap, hub) + ok = await sw._check_recordings_writable() + assert ok is False + assert hub.events == [{ + "type": "sync_error", + "kind": "recordings_unwritable", + "message": "recordings path not writable", + }] + + +async def test_check_recordings_writable_clears_previous_error(tmp_path): + snap = types.SimpleNamespace(recordings=str(tmp_path)) + hub = _RecordingHub() + sw = _make_worker(snap, hub) + # Seed a pretend previous error so we can check clearance. + sw._last_error_kind = "recordings_unwritable" + ok = await sw._check_recordings_writable() + assert ok is True + assert {"type": "sync_error", "kind": None, "message": None} in hub.events + + +async def test_check_recordings_writable_does_not_emit_when_already_clear(tmp_path): + snap = types.SimpleNamespace(recordings=str(tmp_path)) + hub = _RecordingHub() + sw = _make_worker(snap, hub) + sw._last_error_kind = None + ok = await sw._check_recordings_writable() + assert ok is True + # No event of any kind — the path was fine and nothing was wrong before. + assert hub.events == [] + + +async def test_listing_http_401_emits_auth_failure_error(): + import urllib.error + + hub = _RecordingHub() + snap = types.SimpleNamespace(address="cam.local") + sw = _make_worker(snap, hub) + err = urllib.error.HTTPError("http://cam.local/", 401, "Unauthorized", {}, None) + await sw._classify_listing_failure(err) + assert hub.events == [{ + "type": "sync_error", + "kind": "auth_failure", + "message": "camera authentication failed", + }] + + +async def test_listing_non_auth_error_does_not_set_auth_failure(): + import urllib.error + hub = _RecordingHub() + sw = _make_worker(types.SimpleNamespace(), hub) + err = urllib.error.HTTPError("http://cam.local/", 500, "Server error", {}, None) + await sw._classify_listing_failure(err) + # Not an auth failure — no sync_error emitted. + assert hub.events == [] + + +async def test_clear_sync_error_emits_only_when_was_set(): + hub = _RecordingHub() + sw = _make_worker(types.SimpleNamespace(), hub) + sw._last_error_kind = "auth_failure" + await sw._clear_sync_error() + assert hub.events == [{"type": "sync_error", "kind": None, "message": None}] + # Calling it again is a no-op. + hub.events.clear() + await sw._clear_sync_error() + assert hub.events == [] diff --git a/web/app.py b/web/app.py index 041e81a..d646bf1 100644 --- a/web/app.py +++ b/web/app.py @@ -142,7 +142,16 @@ async def _background_retention() -> None: app.state.retention_task = asyncio.create_task(_background_retention()) - app.state.hub = Hub() + app.state.hub = Hub(settings_provider=provider) + # Compute initial sync_status so the very first WebSocket snapshot + # and the first MQTT publish carry the right value without waiting + # for the sync worker's first cycle. + from web.services.sync_status import compute_sync_status as _csss + try: + _state, _ = _csss(app.state.hub, None, provider.get()) + app.state.hub.last_state["sync_status"] = _state + except Exception: + log.exception("initial sync_status compute failed") app.state.geocode = GeocodeService(app.state.db, provider) app.state.export_worker = ExportWorker( app.state.db, provider, app.state.hub.broadcast diff --git a/web/services/hub.py b/web/services/hub.py index 62ca04a..e81bccb 100644 --- a/web/services/hub.py +++ b/web/services/hub.py @@ -22,19 +22,33 @@ from fastapi import WebSocket from starlette.websockets import WebSocketDisconnect +from .sync_status import compute_sync_status + log = logging.getLogger("viofosync.hub") class Hub: - def __init__(self) -> None: + def __init__(self, settings_provider: Any = None) -> None: self._clients: Set[WebSocket] = set() self._lock = asyncio.Lock() + self._settings_provider = settings_provider # Retain the last snapshot of major state so a newly- # connected client sees the current situation without # waiting for the next event. self.last_state: Dict[str, Any] = { "dashcam_online": None, "current_item": None, + # Stateful diagnostics consumed by compute_sync_status(): + "sync_error": None, + "disk_pct": None, + # Latest computed status + reason. Stored on the hub so + # the WebSocket snapshot (and any consumer reading + # last_state) sees a coherent pair without waiting for the + # next change event. ``sync_status_reason`` is the + # human-readable error reason — non-null only when status + # is "error". + "sync_status": None, + "sync_status_reason": None, } async def connect(self, ws: WebSocket) -> None: @@ -91,8 +105,23 @@ async def broadcast(self, event: Dict[str, Any]) -> None: "running": event.get("running"), "paused": event.get("paused"), } + elif t == "sync_error": + # kind=None is the clear signal. Anything else replaces the + # current error verbatim — last writer wins. + kind = event.get("kind") + if kind is None: + self.last_state["sync_error"] = None + else: + self.last_state["sync_error"] = { + "kind": kind, + "message": event.get("message"), + } + elif t == "disk_pct": + pct = event.get("pct") + if isinstance(pct, (int, float)): + self.last_state["disk_pct"] = float(pct) - dead = [] + dead: list = [] async with self._lock: clients = list(self._clients) for ws in clients: @@ -100,11 +129,46 @@ async def broadcast(self, event: Dict[str, Any]) -> None: await ws.send_json(event) except Exception: dead.append(ws) + # Recompute the unified status after the state mutation above. + await self._maybe_emit_sync_status(dead) if dead: async with self._lock: for ws in dead: self._clients.discard(ws) + async def _maybe_emit_sync_status(self, dead: list) -> None: + """After last_state mutations, recompute the unified status. If + it differs from the cached value, store it and broadcast a + follow-up event. Called from broadcast(); ``dead`` is the same + list it accumulates so we drop disconnected clients in one pass. + """ + if self._settings_provider is None: + return + try: + snap = self._settings_provider.get() + status, reason = compute_sync_status(self, None, snap) + except Exception: + log.exception("sync_status compute failed; skipping follow-up") + return + prev_status = self.last_state.get("sync_status") + prev_reason = self.last_state.get("sync_status_reason") + # Dedupe on the (status, reason) pair so a changing reason + # (e.g. disk % climbing while status stays "error") still + # reaches clients. Non-error states have reason=None so this + # collapses back to status-only deduping there. + if status == prev_status and reason == prev_reason: + return + self.last_state["sync_status"] = status + self.last_state["sync_status_reason"] = reason + event = {"type": "sync_status", "status": status, "reason": reason} + async with self._lock: + clients = list(self._clients) + for ws in clients: + try: + await ws.send_json(event) + except Exception: + dead.append(ws) + def schedule_broadcast( self, loop: asyncio.AbstractEventLoop, diff --git a/web/services/mqtt.py b/web/services/mqtt.py index 24df56b..46f12e7 100644 --- a/web/services/mqtt.py +++ b/web/services/mqtt.py @@ -109,6 +109,28 @@ def entities_affected_by(hub_event_type: str): yield entity +async def _publish_entity_attrs(client, cfg, entity, hub, db, snap, + *, maybe_publish): + """Publish the JSON attributes payload for an entity that defines + ``attrs_fn``. No-op when attrs_fn is None or returns None.""" + if entity.attrs_fn is None: + return + try: + attrs = entity.attrs_fn(hub, db, snap) + except Exception: + log.exception("mqtt: attrs_fn raised for %s", entity.object_id) + return + if attrs is None: + return + from .mqtt_topology import build_attrs_topic + import json as _json + topic = build_attrs_topic(entity.object_id, cfg) + payload = _json.dumps(attrs).encode() + await maybe_publish(client, topic, payload, + retain=True, qos=cfg["qos"], + min_interval=entity.min_publish_interval_s) + + class ConnState(enum.Enum): IDLE = "idle" # service not started or settings incomplete CONNECTING = "connecting" @@ -309,6 +331,10 @@ async def _publish_full_state(self, client, cfg: dict) -> None: await self._maybe_publish(client, topic, value.encode(), retain=True, qos=cfg["qos"], min_interval=entity.min_publish_interval_s) + await _publish_entity_attrs( + client, cfg, entity, self._hub, self._db, snap, + maybe_publish=self._maybe_publish, + ) async def _maybe_publish( self, client, topic: str, payload: bytes, @@ -389,6 +415,10 @@ def _intercepting_schedule(running_loop, event: dict) -> None: qos=cfg["qos"], min_interval=entity.min_publish_interval_s, ) + await _publish_entity_attrs( + client, cfg, entity, self._hub, self._db, snap, + maybe_publish=self._maybe_publish, + ) finally: # Restore original broadcast methods so subsequent test # runs / restarts don't accumulate interceptors. diff --git a/web/services/mqtt_state.py b/web/services/mqtt_state.py index 2991310..fb5f19a 100644 --- a/web/services/mqtt_state.py +++ b/web/services/mqtt_state.py @@ -15,6 +15,8 @@ import datetime as _dt from typing import Any, Optional +from .sync_status import compute_sync_status + def _iso_z(ts: int) -> str: """ISO 8601 with explicit UTC marker — what HA's timestamp @@ -37,26 +39,17 @@ def state_dashcam(hub, db, snapshot) -> Optional[str]: def state_sync_status(hub, db, snapshot) -> Optional[str]: - sync_state = hub.last_state.get("sync_state") - current_item = hub.last_state.get("current_item") - if not sync_state or not sync_state.get("running"): - return "stopped" - if sync_state.get("paused"): - return "paused" - if current_item: - return "downloading" - # Between items in a sync cycle the in-flight slot is briefly empty - # while the worker writes the GPX sidecar, marks the row done, and - # picks the next clip. Treat "queue still has unfinished work" as - # downloading so HA doesn't flicker to "idle" every few seconds. - with db.conn() as c: - row = c.execute( - "SELECT COUNT(*) AS n FROM download_queue " - "WHERE state IN ('pending', 'downloading')" - ).fetchone() - if row["n"] > 0: - return "downloading" - return "idle" + """The four-state unified status string. See sync_status.py.""" + state, _reason = compute_sync_status(hub, db, snapshot) + return state + + +def attrs_sync_status(hub, db, snapshot) -> Optional[dict]: + """JSON attributes payload for the sync_status sensor. Always + returns a dict with a ``reason`` key so HA templating doesn't have + to guard for a missing attribute.""" + _state, reason = compute_sync_status(hub, db, snapshot) + return {"reason": reason} # ---- queue counts diff --git a/web/services/mqtt_topology.py b/web/services/mqtt_topology.py index 4f33ad5..00098d4 100644 --- a/web/services/mqtt_topology.py +++ b/web/services/mqtt_topology.py @@ -41,6 +41,7 @@ class EntityDef: state_fn: Optional[Callable] # signature pinned in Task 6 command_handler: Optional[Callable[[bytes], Awaitable[None]]] affected_by_hub_events: tuple[str, ...] = field(default_factory=tuple) + attrs_fn: Optional[Callable] = None def build_state_topic(object_id: str, cfg: dict) -> str: @@ -51,6 +52,10 @@ def build_command_topic(object_id: str, cfg: dict) -> str: return f"{cfg['node_id']}/{object_id}/cmd" +def build_attrs_topic(object_id: str, cfg: dict) -> str: + return f"{cfg['node_id']}/{object_id}/attr" + + def build_discovery_topic(component: str, object_id: str, cfg: dict) -> str: return ( f"{cfg['discovery_prefix']}/{component}/" @@ -95,6 +100,8 @@ def build_discovery_payload(entity: EntityDef, cfg: dict) -> dict: } if entity.component in ("sensor", "binary_sensor"): payload["state_topic"] = build_state_topic(entity.object_id, cfg) + if entity.component in ("sensor", "binary_sensor") and entity.attrs_fn is not None: + payload["json_attributes_topic"] = build_attrs_topic(entity.object_id, cfg) if entity.component == "button": payload["command_topic"] = build_command_topic(entity.object_id, cfg) if entity.icon: @@ -139,8 +146,12 @@ def build_discovery_payload(entity: EntityDef, cfg: dict) -> dict: state_fn=_st.state_sync_status, command_handler=None, affected_by_hub_events=( - "sync_state", "item_started", "item_finished", "queue_changed", + "sync_state", "item_started", "item_finished", + "queue_changed", + "dashcam_online", "dashcam_offline", + "disk_pct", "sync_error", ), + attrs_fn=_st.attrs_sync_status, ), # --- Queue --- diff --git a/web/services/retention.py b/web/services/retention.py index 6c626a2..9092a89 100644 --- a/web/services/retention.py +++ b/web/services/retention.py @@ -247,6 +247,12 @@ def disk_used_pct(recordings: str, quota_gb: int = 0) -> Optional[float]: ``_quota_exceeded`` to decide whether each rule is currently breached, so a quota set without a percentage threshold (or vice versa) triggers independently. + + NOTE: For deciding whether the disk is critically full (the + ``compute_sync_status`` error trigger), prefer + :func:`filesystem_used_pct` — quota mode here reads ~100% when + retention is doing its job correctly, which would otherwise trip + a perpetual error. """ if quota_gb > 0: used = _cached_used_bytes(recordings) @@ -254,6 +260,17 @@ def disk_used_pct(recordings: str, quota_gb: int = 0) -> Optional[float]: if limit <= 0: return None return used / limit * 100.0 + return filesystem_used_pct(recordings) + + +def filesystem_used_pct(recordings: str) -> Optional[float]: + """Filesystem-level disk usage % for the volume holding *recordings*. + Ignores any configured quota — this is the "OS will start denying + writes soon" signal, separate from the self-imposed quota that + retention manages. + + Returns ``None`` when the path is missing. + """ try: du = shutil.disk_usage(recordings) except (OSError, FileNotFoundError): diff --git a/web/services/sync_status.py b/web/services/sync_status.py new file mode 100644 index 0000000..0629549 --- /dev/null +++ b/web/services/sync_status.py @@ -0,0 +1,79 @@ +"""Single source of truth for the four-state sync status. + +State precedence (highest wins): error > paused > downloading > waiting. + +Inputs come from ``hub.last_state`` (populated by the Hub.broadcast +side-effects in ``hub.py``) and from the settings ``Snapshot``. + +The function is pure and total — it never raises. Any unexpected +``last_state`` shape resolves to ``waiting`` so the UI cannot blank out. +""" +from __future__ import annotations + +from typing import Any, Optional, Tuple + + +STATES = ("downloading", "waiting", "paused", "error") + + +def compute_sync_status(hub, db, snapshot) -> Tuple[str, Optional[str]]: + """Return ``(state, reason)``. + + ``state`` is one of ``STATES``. ``reason`` is a short human-readable + string when ``state == "error"``, else ``None``. + + ``db`` is accepted for signature symmetry with ``state_fn`` callers + in mqtt_state.py; not used today. + """ + try: + return _compute(hub, snapshot) + except Exception: + # Defensive: any unexpected exception falls back to waiting. + # The UI is rendered from this value; never let it crash. + return "waiting", None + + +def _compute(hub, snapshot) -> Tuple[str, Optional[str]]: + last = getattr(hub, "last_state", None) or {} + + # ---- error tier (highest precedence) ---- + + # Missing required config — sticky, takes priority over everything. + if not getattr(snapshot, "address", None): + return "error", "camera address not configured" + + # Stateful sync_error captured by the worker (recordings unwritable, + # auth failure, etc.). Use its message verbatim as the reason. + sync_error = last.get("sync_error") + if isinstance(sync_error, dict): + msg = sync_error.get("message") or "sync error" + return "error", str(msg) + + # Disk-pressure error. ``disk_critical_pct == 0`` disables the check. + critical = int(getattr(snapshot, "disk_critical_pct", 0) or 0) + disk_pct = last.get("disk_pct") + if critical > 0 and isinstance(disk_pct, (int, float)) and disk_pct >= critical: + return "error", f"disk {round(disk_pct)}% full" + + # ---- paused tier ---- + + sync_state = last.get("sync_state") + if not isinstance(sync_state, dict): + # Worker hasn't reported in yet — show paused rather than waiting, + # because nothing is happening from the user's perspective. + return "paused", None + if not sync_state.get("running"): + return "paused", None + if sync_state.get("paused"): + return "paused", None + + # ---- downloading ---- + + current_item = last.get("current_item") + dashcam_online = last.get("dashcam_online") + if current_item and dashcam_online is True: + return "downloading", None + + # ---- waiting (default for running-but-not-downloading) ---- + + return "waiting", None diff --git a/web/services/sync_worker.py b/web/services/sync_worker.py index 001b822..cdb35ed 100644 --- a/web/services/sync_worker.py +++ b/web/services/sync_worker.py @@ -36,6 +36,7 @@ from ..db import Database from ..settings import SettingsProvider from . import queue as q +from . import retention as _retention from . import scanner from .hub import Hub @@ -278,6 +279,10 @@ def __init__( self._loop: Optional[asyncio.AbstractEventLoop] = None self._running_cycle = False self._current_filename: Optional[str] = None + # Tracks the kind of sync_error currently sticky on the hub, so + # we can emit clear signals only when a previously-set error + # actually changes. + self._last_error_kind: Optional[str] = None # ---- lifecycle ---- @@ -394,6 +399,71 @@ def _broadcast_sync_state(self) -> None: ) ) + async def _emit_disk_pct(self) -> None: + """Broadcast the current filesystem disk usage % so the Hub + can evaluate the critical-disk error condition. Called from + ``_cycle`` on the event loop. + + Uses :func:`retention.filesystem_used_pct` — NOT the + quota-aware ``disk_used_pct``. When ``RECORDINGS_QUOTA_GB`` + is set, quota retention deliberately keeps the recordings dir + at ~100% of quota, so the quota-aware metric would trip + ``DISK_CRITICAL_PCT`` perpetually. The critical-disk error + is reserved for the filesystem-level "OS will deny writes + soon" condition, which is independent of our self-imposed + quota. + """ + snap = self._provider.get() + pct = _retention.filesystem_used_pct(snap.recordings) + if pct is None: + return + await self.hub.broadcast({"type": "disk_pct", "pct": float(pct)}) + + async def _emit_sync_error( + self, kind: Optional[str], message: Optional[str] + ) -> None: + await self.hub.broadcast({ + "type": "sync_error", "kind": kind, "message": message, + }) + + async def _set_sync_error(self, kind: str, message: str) -> None: + if self._last_error_kind == kind: + return # already sticky on the hub; no event + self._last_error_kind = kind + await self._emit_sync_error(kind, message) + + async def _clear_sync_error(self) -> None: + if self._last_error_kind is None: + return + self._last_error_kind = None + await self._emit_sync_error(None, None) + + async def _check_recordings_writable(self) -> bool: + """Return True if the recordings root exists and is writable. + Emits a sticky sync_error on failure, clears one on recovery.""" + snap = self._provider.get() + path = getattr(snap, "recordings", None) or "" + ok = bool(path) and os.path.isdir(path) and os.access(path, os.W_OK) + if not ok: + await self._set_sync_error( + "recordings_unwritable", + "recordings path not writable", + ) + return False + if self._last_error_kind == "recordings_unwritable": + await self._clear_sync_error() + return True + + async def _classify_listing_failure(self, exc: BaseException) -> None: + """Inspect a listing exception. If it's HTTP 401/403, emit a + sticky auth_failure error. Other exceptions are not promoted to + sticky errors — they're transient by nature.""" + if isinstance(exc, urllib.error.HTTPError) and exc.code in (401, 403): + await self._set_sync_error( + "auth_failure", + "camera authentication failed", + ) + # ---- main loop ---- async def _run(self) -> None: @@ -465,6 +535,7 @@ async def _refresh_listing_and_reconcile(self) -> bool: ) except Exception as e: log.warning("listing fetch failed: %s", e) + await self._classify_listing_failure(e) return False if self._provider.get().sync_ro_only: listing = list(_filter_ro_only(listing)) @@ -476,9 +547,14 @@ async def _refresh_listing_and_reconcile(self) -> bool: "queue": q.list_all(self.db, limit=200), }) q.emit_queue_changed(self.db, self.hub) + if self._last_error_kind == "auth_failure": + await self._clear_sync_error() return True async def _cycle(self) -> bool: + await self._emit_disk_pct() + if not await self._check_recordings_writable(): + return False reachable = await self._probe() await self.hub.broadcast({ "type": "dashcam_online" if reachable else "dashcam_offline", @@ -546,7 +622,6 @@ async def _cycle(self) -> bool: await scanner.sweep_missing_thumbs( self.db, snap.recordings, ) - from . import retention as _retention sink = WebSink(self.hub, asyncio.get_running_loop()) await asyncio.to_thread( _retention.sweep, @@ -560,6 +635,7 @@ async def _cycle(self) -> bool: except Exception: # pragma: no cover — non-fatal log.exception("post-cycle scan/thumb sweep failed") + await self._emit_disk_pct() await self.hub.broadcast({ "type": "sync_done", "ok": True, diff --git a/web/settings.py b/web/settings.py index 2df8ab8..26b1c73 100644 --- a/web/settings.py +++ b/web/settings.py @@ -56,6 +56,7 @@ class Snapshot: retention_disk_pct: int retention_protect_ro: bool recordings_quota_gb: int + disk_critical_pct: int password_hash: str session_secret: str @@ -237,6 +238,7 @@ def _make_snapshot(self, data: dict) -> Snapshot: retention_disk_pct=m.RETENTION_DISK_PCT, retention_protect_ro=m.RETENTION_PROTECT_RO, recordings_quota_gb=m.RECORDINGS_QUOTA_GB, + disk_critical_pct=m.DISK_CRITICAL_PCT, password_hash=m.WEB_PASSWORD_HASH, session_secret=m.SESSION_SECRET, host=m.WEB_HOST, diff --git a/web/settings_schema.py b/web/settings_schema.py index d8b304e..036e980 100644 --- a/web/settings_schema.py +++ b/web/settings_schema.py @@ -50,6 +50,10 @@ class SettingsModel(BaseModel): # share (Synology shared folder, ZFS dataset quota, etc.) where the # OS-level "free space" doesn't reflect the actual constraint. RECORDINGS_QUOTA_GB: int = Field(default=0, ge=0, le=1_048_576) + # When sync sees disk usage >= this percentage, status flips to + # "error" with reason "disk N% full". Must be >= RETENTION_DISK_PCT + # so retention gets a chance to clean before we flag a critical state. + DISK_CRITICAL_PCT: int = Field(default=95, ge=0, le=100) WEB_HOST: str = "0.0.0.0" WEB_PORT: int = Field(default=8080, ge=1, le=65535) @@ -131,6 +135,16 @@ def _validate_mqtt_cross_field(self): raise ValueError("MQTT_HOST is required when MQTT_ENABLED is True") return self + @model_validator(mode="after") + def _validate_disk_critical(self): + # Allow 0 (disabled) regardless. Otherwise must be >= retention pct. + if self.DISK_CRITICAL_PCT != 0 and self.RETENTION_DISK_PCT > self.DISK_CRITICAL_PCT: + raise ValueError( + f"DISK_CRITICAL_PCT ({self.DISK_CRITICAL_PCT}) must be " + f">= RETENTION_DISK_PCT ({self.RETENTION_DISK_PCT})" + ) + return self + # Public taxonomy used by the API + UI. EDITABLE_KEYS = { @@ -139,7 +153,7 @@ def _validate_mqtt_cross_field(self): "ENABLE_SCHEDULED_SYNC", "WEB_HOST", "WEB_PORT", "EXPORT_ENCODER", "NOMINATIM_EMAIL", "GEOCODE_ENABLED", "SYNC_RO_ONLY", "RETENTION_MAX_DAYS", "RETENTION_DISK_PCT", - "RETENTION_PROTECT_RO", "RECORDINGS_QUOTA_GB", + "RETENTION_PROTECT_RO", "RECORDINGS_QUOTA_GB", "DISK_CRITICAL_PCT", "DISTANCE_UNITS", "PIP_POSITION", "MQTT_ENABLED", "MQTT_HOST", "MQTT_PORT", "MQTT_USERNAME", diff --git a/web/static/app.js b/web/static/app.js index b72a7db..37f040a 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -86,7 +86,10 @@ async function showApp() { openSocket(); try { const s = await api("/api/sync/status"); - updateSyncState(s.running, s.paused); + state.syncRunning = s.running; + state.syncPaused = s.paused; + // The WS snapshot will deliver the server-computed sync_status + // shortly; no direct updateSyncState call needed here. } catch {} try { const gs = await api("/api/archive/extract-gps/status"); @@ -161,21 +164,21 @@ document.getElementById("sync-toggle").addEventListener("click", async () => { } else { await api("/api/sync/resume", { method: "POST" }); } - // Fetch fresh status — the WS event will follow shortly but - // a direct response avoids a visible lag on the button icon. const s = await api("/api/sync/status"); - updateSyncState(s.running, s.paused); + state.syncRunning = s.running; + state.syncPaused = s.paused; + // The WS sync_status event will follow shortly; no direct call to + // updateSyncState here — the server-computed status is the truth. }); -function updateSyncState(running, paused) { - state.syncRunning = running; - state.syncPaused = paused; - +// state.syncStatus is one of: "downloading", "waiting", "paused", "error", null +// state.syncStatusReason is a short human-readable string (only set in error) +function updateSyncState(status) { + state.syncStatus = status; + // The toggle button's pause/resume behaviour still depends on + // syncRunning/syncPaused, which are derived from sync_state events + // (see handleEvent). Status is what drives the *visual* badge + icon. const btn = document.getElementById("sync-toggle"); - // Use explicit setAttribute/removeAttribute on the SVG icons: - // some browsers don't propagate the `.hidden` IDL property - // setter onto SVGElement reliably, which has bitten us with - // the icon staying visible when JS said it should be hidden. const setVisible = (el, visible) => { if (visible) el.removeAttribute("hidden"); else el.setAttribute("hidden", ""); @@ -183,26 +186,33 @@ function updateSyncState(running, paused) { const iconPlay = document.getElementById("sync-icon-play"); const iconPause = document.getElementById("sync-icon-pause"); const iconSync = document.getElementById("sync-icon-sync"); + const iconWarn = document.getElementById("sync-icon-warning"); let show, title, klass; - if (!running) { - show = iconPlay; - title = "Start downloading"; - klass = null; - } else if (paused) { - show = iconPause; - title = "Resume downloading"; - klass = "paused"; + if (status === "downloading") { + show = iconSync; title = "Pause downloading"; klass = "active"; + } else if (status === "waiting") { + show = iconSync; title = "Pause downloading"; klass = "waiting"; + } else if (status === "paused") { + show = iconPause; title = "Resume downloading"; klass = "paused"; + } else if (status === "error") { + // Surface the reason on the button tooltip too — users hovering + // the icon (not the badge) should still see why we're in error. + show = iconWarn; + title = state.syncStatusReason + ? "Error: " + state.syncStatusReason + : "Error"; + klass = "error"; } else { - show = iconSync; - title = "Pause downloading"; - klass = "active"; + // Unknown / initial — fall back to play icon, no class. + show = iconPlay; title = "Start downloading"; klass = null; } setVisible(iconPlay, show === iconPlay); setVisible(iconPause, show === iconPause); setVisible(iconSync, show === iconSync); - btn.classList.remove("active", "paused"); + setVisible(iconWarn, show === iconWarn); + btn.classList.remove("active", "paused", "waiting", "error"); if (klass) btn.classList.add(klass); btn.title = title; } @@ -1683,31 +1693,51 @@ function appendLog(ev) { function handleEvent(ev) { appendLog(ev); - const statusEl = document.getElementById("dashcam-status"); + const statusEl = document.getElementById("sync-status"); + const STATUS_LABEL = { + downloading: "Downloading", + waiting: "Waiting", + paused: "Paused", + error: "Error", + }; + const applyStatus = (status, reason) => { + state.syncStatus = status; + state.syncStatusReason = reason || null; + // For error states, surface the reason directly in the badge text + // rather than burying it in a hover-only tooltip — users won't + // know to hover, and on touch devices the tooltip is invisible. + let badgeText = STATUS_LABEL[status] || ""; + if (status === "error" && reason) { + badgeText = "Error: " + reason; + } + statusEl.textContent = badgeText; + statusEl.className = "status " + (status || ""); + statusEl.title = reason || ""; + updateSyncState(status); + }; switch (ev.type) { case "snapshot": - if (ev.state.dashcam_online === true) { - statusEl.textContent = "Dashcam online"; - statusEl.className = "status online"; - } else if (ev.state.dashcam_online === false) { - statusEl.textContent = "Dashcam offline"; - statusEl.className = "status offline"; + if (ev.state.sync_status) { + applyStatus(ev.state.sync_status, ev.state.sync_status_reason); } if (ev.state.current_item) updateCurrent(ev.state.current_item); if (ev.state.sync_state) { - updateSyncState(ev.state.sync_state.running, ev.state.sync_state.paused); + state.syncRunning = ev.state.sync_state.running; + state.syncPaused = ev.state.sync_state.paused; } break; + case "sync_status": + applyStatus(ev.status, ev.reason); + break; case "sync_state": - updateSyncState(ev.running, ev.paused); + state.syncRunning = ev.running; + state.syncPaused = ev.paused; + // Status follow-up will arrive separately; don't drive the badge here. break; case "dashcam_online": - statusEl.textContent = "Dashcam online"; - statusEl.className = "status online"; - break; case "dashcam_offline": - statusEl.textContent = "Dashcam offline — retrying…"; - statusEl.className = "status offline"; + // Connection events are still logged but no longer drive the badge. + // The follow-up sync_status event handles UI updates. break; case "item_started": updateCurrent({ filename: ev.filename, total: ev.total, bytes: 0 }); diff --git a/web/static/index.html b/web/static/index.html index a33280c..ac91fab 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -35,7 +35,7 @@

Viofosync

Downloads
- + diff --git a/web/static/styles.css b/web/static/styles.css index 4ea784e..ef8d988 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -155,6 +155,20 @@ input[type="radio"] { from { transform: rotate(0deg); } to { transform: rotate(360deg); } } +/* In "waiting" state we still show the sync icon, but static. The + * .active class drives the spin; .waiting suppresses it. */ +#sync-toggle.waiting #sync-icon-sync { + animation: none; +} +#sync-toggle.error #sync-icon-warning { + /* Subtle pulse on the warning icon so the user notices a fault state + * without it being too aggressive. */ + animation: warn-pulse 2s ease-in-out infinite; +} +@keyframes warn-pulse { + 0%, 100% { opacity: 0.9; } + 50% { opacity: 0.6; } +} .icon-btn.paused { color: var(--err-text); border-color: var(--err); @@ -247,8 +261,17 @@ nav a.active { background: var(--panel-2); } border: 1px solid var(--border); font-size: 12px; } -.status.offline { color: var(--err-text); border-color: var(--err); } -.status.online { color: var(--ok); border-color: var(--ok); } +/* Four-state sync status badge. Colors reuse the existing palette so + * dark/light themes stay coherent — extend the palette only if a + * future revision adds new accent tokens. */ +.status.downloading { color: var(--ok); border-color: var(--ok); } +.status.waiting { color: #f59e0b; border-color: #f59e0b; } +.status.paused { color: var(--err-text); border-color: var(--err); } +.status.error { + color: #ffffff; + background: #7f1d1d; + border-color: #7f1d1d; +} main { padding: 24px; @@ -1374,8 +1397,10 @@ main { background: var(--muted); border: none; } - .status.offline { background: var(--err); } - .status.online { background: var(--ok); } + .status.downloading { background: var(--ok); } + .status.waiting { background: #f59e0b; } + .status.paused { background: var(--err); } + .status.error { background: #7f1d1d; } .day-header h3 { font-size: 14px; } .day-header .meta { font-size: 11px; } @@ -1461,6 +1486,10 @@ main { animation: none; box-shadow: 0 0 0 1px var(--err); } + #sync-toggle.error #sync-icon-warning { + animation: none; + opacity: 0.9; + } .clip-pair.flash { animation: none; } .exports-panel.just-submitted { animation: none; } .skip-link { transition: none; } From dda990ace99b841731d33eca51c1e1f72d067124 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 28 May 2026 23:23:31 +0100 Subject: [PATCH 05/21] Fix MQTT coalescer KeyError race and reduce progress publish load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two production bugs both observable as the HA entity briefly going Unavailable when the MqttService trips its reconnect cycle. 1. PublishCoalescer race (KeyError: 'viofosync/current_progress/state') flush_due() snapshots _pending, then awaits sink() per topic, then does `del self._pending[topic]`. During the sink await, a concurrent consider() from _drain_publishes can run; if its payload equals the not-yet-updated _last_payload[topic] it takes the first branch and pops the same entry. The trailing del then raises KeyError, crashes the tick task, and trips the reconnect cycle. A concurrent consider could also replace the entry with a NEWER pending which the bare del silently dropped — a latent data-loss bug paired with the KeyError. Fix: identity check after the await in both flush_due() and consider()'s immediate-publish branch. Only clear _pending[topic] if it still references the same object we just published; otherwise leave whatever a concurrent task installed in place. 2. aiomqtt publish timeout on high-frequency current_progress topic The `current_progress` entity fires on every `item_progress` event during a download. QoS=1 PUBACK waits stall the publisher under broker latency and grow the in-flight queue; aiomqtt eventually times out the publish and the connection drops. The retained value in HA means losing a single progress update is acceptable. Fix: optional per-entity qos override on EntityDef (defaults to the global MQTT_QOS setting). Set qos=0 on current_progress only. All four publish sites in MqttService now resolve to entity.qos when set, falling back to cfg["qos"]. Other entities (sync_status, dashcam, last_downloaded_clip, disk_used, queue_*) keep QoS=1 for reliable state delivery. Tests: - test_flush_due_survives_concurrent_consider_popping_pending — deterministic reproduction of the KeyError race using a blocking sink to interleave a payload-matches-last_payload consider(). - test_flush_due_preserves_newer_pending_added_during_await — deterministic reproduction of the data-loss companion bug. - test_current_progress_uses_qos_0 / test_state_entities_default_to_global_qos structural assertions on the topology change. --- tests/test_mqtt_publication.py | 112 +++++++++++++++++++++++++++++++++ tests/test_mqtt_topology.py | 22 +++++++ web/services/mqtt.py | 28 +++++++-- web/services/mqtt_topology.py | 5 ++ 4 files changed, 161 insertions(+), 6 deletions(-) diff --git a/tests/test_mqtt_publication.py b/tests/test_mqtt_publication.py index 3273a48..f366d51 100644 --- a/tests/test_mqtt_publication.py +++ b/tests/test_mqtt_publication.py @@ -117,3 +117,115 @@ async def sink(topic, payload, retain, qos): now[0] = 6.0 await pc.flush_due(sink) assert sent == [b"1"] # only the original publish + + +@pytest.mark.asyncio +async def test_flush_due_survives_concurrent_consider_popping_pending(): + """Race regression: flush_due was using `del self._pending[topic]` + after `await sink(...)`. A concurrent `consider()` whose payload + equals the still-unchanged `_last_payload[topic]` pops the same + entry during the yield, and the trailing del raises KeyError — + crashing the tick task and triggering an MQTT reconnect. + """ + from web.services.mqtt import PublishCoalescer + now = [0.0] + pc = PublishCoalescer(monotonic=lambda: now[0]) + + # Get topic X into "published once" state with last_payload = b"A". + sent: list[tuple[str, bytes]] = [] + async def fast_sink(topic, payload, retain, qos): + sent.append((topic, payload)) + await pc.consider("X", b"A", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + + # Within cooldown: stash a new pending value b"B". + now[0] = 1.0 + await pc.consider("X", b"B", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + + # Cooldown elapsed. Trigger flush_due against a sink that blocks + # so we can interleave another consider() call mid-publish. + now[0] = 10.0 + publish_started = asyncio.Event() + publish_release = asyncio.Event() + + async def slow_sink(topic, payload, retain, qos): + publish_started.set() + await publish_release.wait() + sent.append((topic, payload)) + + flush_task = asyncio.create_task(pc.flush_due(slow_sink)) + await publish_started.wait() # flush_due is now mid-await + + # During the yield, _last_payload[X] is still b"A" (flush_due hasn't + # updated it yet). A consider() with payload b"A" therefore matches + # the first branch of consider and pops _pending[X]. + await pc.consider("X", b"A", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + + publish_release.set() + # Pre-fix this raised KeyError: 'X' from the trailing `del`. + await flush_task + + # The flushed b"B" must still have been recorded as the latest + # published payload (so subsequent identical considers are suppressed). + assert pc._last_payload["X"] == b"B" + assert "X" not in pc._pending + + +@pytest.mark.asyncio +async def test_flush_due_preserves_newer_pending_added_during_await(): + """Latent data-loss bug paired with the KeyError race: if a concurrent + consider() stashes a NEWER pending entry while flush_due is awaiting + sink, the trailing `del self._pending[topic]` would drop that newer + entry. The newer pending must survive so it can be flushed next tick. + + Forcing the stash path requires two sequential considers during the + slow await: the first hits the immediate-publish branch and updates + _last_publish[X] to `now`, so the second sees the cooldown as + unelapsed and stashes. + """ + from web.services.mqtt import PublishCoalescer + now = [0.0] + pc = PublishCoalescer(monotonic=lambda: now[0]) + + sent: list[tuple[str, bytes]] = [] + async def fast_sink(topic, payload, retain, qos): + sent.append((topic, payload)) + + await pc.consider("X", b"A", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + now[0] = 1.0 + await pc.consider("X", b"B", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + + now[0] = 10.0 + publish_started = asyncio.Event() + publish_release = asyncio.Event() + + async def slow_sink(topic, payload, retain, qos): + publish_started.set() + await publish_release.wait() + sent.append((topic, payload)) + + flush_task = asyncio.create_task(pc.flush_due(slow_sink)) + await publish_started.wait() + + # First consider during the yield: cooldown is elapsed (last_publish + # is still 0.0), takes immediate-publish path, sets _last_publish[X] + # = 10.0 and pops the b"B" pending entry. + await pc.consider("X", b"C", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + # Second consider: cooldown now NOT elapsed (10 - 10 < 5), so stash + # branch — installs b"D" as the new pending. + await pc.consider("X", b"D", min_interval=5.0, + sink=fast_sink, retain=True, qos=1) + + publish_release.set() + await flush_task + + # b"D" must survive — flush_due must not delete a pending entry it + # didn't install. Pre-fix, `del self._pending[topic]` silently + # dropped it. + assert "X" in pc._pending, "newer pending entry was dropped by flush_due" + assert pc._pending["X"].payload == b"D" diff --git a/tests/test_mqtt_topology.py b/tests/test_mqtt_topology.py index 0dbb306..8811c05 100644 --- a/tests/test_mqtt_topology.py +++ b/tests/test_mqtt_topology.py @@ -114,3 +114,25 @@ def test_sync_status_entity_has_attrs_fn(): from web.services.mqtt_state import attrs_sync_status entity = next(e for e in TOPOLOGY if e.object_id == "sync_status") assert entity.attrs_fn is attrs_sync_status + + +def test_current_progress_uses_qos_0(): + """current_progress fires on every item_progress event during a + download — it's the highest-rate publisher. QoS=1 PUBACK waits + stall the publisher under broker latency and trip the connection. + Retained QoS=0 still lets HA pick up the latest value on subscribe; + losing a single progress update mid-flight is acceptable. + """ + from web.services.mqtt_topology import TOPOLOGY + entity = next(e for e in TOPOLOGY if e.object_id == "current_progress") + assert entity.qos == 0 + + +def test_state_entities_default_to_global_qos(): + """Entities without an explicit qos override fall back to cfg['qos']. + Verify sync_status / dashcam (the reliability-sensitive state + entities) leave qos unset so the global setting wins.""" + from web.services.mqtt_topology import TOPOLOGY + by_id = {e.object_id: e for e in TOPOLOGY} + assert by_id["sync_status"].qos is None + assert by_id["dashcam"].qos is None diff --git a/web/services/mqtt.py b/web/services/mqtt.py index 46f12e7..1191d22 100644 --- a/web/services/mqtt.py +++ b/web/services/mqtt.py @@ -62,10 +62,17 @@ async def consider( now = self._mono() last = self._last_publish.get(topic) if last is None or (now - last) >= min_interval: + # Snapshot the current pending entry (if any) so that, after + # the sink await yields, we only clear an entry that's still + # the same object. A concurrent flush_due or consider may + # have installed a NEWER pending while we awaited — dropping + # it here would lose data. + pending_before = self._pending.get(topic) await sink(topic, payload, retain, qos) self._last_payload[topic] = payload self._last_publish[topic] = now - self._pending.pop(topic, None) + if self._pending.get(topic) is pending_before: + self._pending.pop(topic, None) return # Still inside the cooldown — stash the latest value. self._pending[topic] = _Pending( @@ -84,7 +91,12 @@ async def flush_due(self, sink: Sink) -> None: await sink(topic, pend.payload, pend.retain, pend.qos) self._last_payload[topic] = pend.payload self._last_publish[topic] = now - del self._pending[topic] + # Identity check: a concurrent consider() running during the + # sink await may have popped this entry (payload matched + # the not-yet-updated _last_payload) — KeyError on bare del — + # or replaced it with a newer pending we must preserve. + if self._pending.get(topic) is pend: + del self._pending[topic] def forget(self, topic: str) -> None: self._last_payload.pop(topic, None) @@ -126,8 +138,9 @@ async def _publish_entity_attrs(client, cfg, entity, hub, db, snap, import json as _json topic = build_attrs_topic(entity.object_id, cfg) payload = _json.dumps(attrs).encode() + qos = entity.qos if entity.qos is not None else cfg["qos"] await maybe_publish(client, topic, payload, - retain=True, qos=cfg["qos"], + retain=True, qos=qos, min_interval=entity.min_publish_interval_s) @@ -328,8 +341,9 @@ async def _publish_full_state(self, client, cfg: dict) -> None: if value is None: continue topic = build_state_topic(entity.object_id, cfg) + qos = entity.qos if entity.qos is not None else cfg["qos"] await self._maybe_publish(client, topic, value.encode(), - retain=True, qos=cfg["qos"], + retain=True, qos=qos, min_interval=entity.min_publish_interval_s) await _publish_entity_attrs( client, cfg, entity, self._hub, self._db, snap, @@ -407,12 +421,13 @@ def _intercepting_schedule(running_loop, event: dict) -> None: continue if value is None: continue + qos = entity.qos if entity.qos is not None else cfg["qos"] await self._maybe_publish( client, build_state_topic(entity.object_id, cfg), value.encode(), retain=True, - qos=cfg["qos"], + qos=qos, min_interval=entity.min_publish_interval_s, ) await _publish_entity_attrs( @@ -544,10 +559,11 @@ async def _sink(t, p, r, q): continue if value is None: continue + qos = entity.qos if entity.qos is not None else cfg["qos"] await self._maybe_publish( client, build_state_topic(entity.object_id, cfg), value.encode(), - retain=True, qos=cfg["qos"], + retain=True, qos=qos, min_interval=0.0, ) diff --git a/web/services/mqtt_topology.py b/web/services/mqtt_topology.py index 00098d4..0202fee 100644 --- a/web/services/mqtt_topology.py +++ b/web/services/mqtt_topology.py @@ -42,6 +42,10 @@ class EntityDef: command_handler: Optional[Callable[[bytes], Awaitable[None]]] affected_by_hub_events: tuple[str, ...] = field(default_factory=tuple) attrs_fn: Optional[Callable] = None + # None → use the global MQTT_QOS setting. Override on disposable + # high-rate entities (e.g. current_progress) to QoS=0 so PUBACK + # latency from the broker can't stall the publisher. + qos: Optional[int] = None def build_state_topic(object_id: str, cfg: dict) -> str: @@ -259,6 +263,7 @@ def build_discovery_payload(entity: EntityDef, cfg: dict) -> dict: command_handler=None, affected_by_hub_events=("item_progress", "item_started", "item_finished"), + qos=0, ), # --- Disk / sync history --- From b79891481961ba8a16a9d2fc88563fa1c1086403 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 4 Jun 2026 14:22:05 +0100 Subject: [PATCH 06/21] Add session download speed + ETA (UI) and Home Assistant speed sensor When a sync session is downloading, the download manager now shows a session-wide moving-average download speed and an estimated time to complete, and a throttled Home Assistant sensor reports the speed. - DownloadSession tracker (web/services/download_session.py): monotonic wire-byte accounting (clamps retry/file-boundary resets), a 30s windowed moving-average speed, and an ETA computed from the pending queue bytes plus the in-flight file remainder. Pure/injectable. - queue.pending_bytes(): sums remote_size of pending rows for the ETA. - Hub feeds the tracker from broadcast() and emits a deduped session_stats follow-up (whole-second elapsed in the key gives a ~1/s UI heartbeat); last_state["session"] rehydrates reconnecting clients. - HA download_speed sensor (data_rate, MB/s, enabled by default): state_download_speed gates publishing until 30s into a session then the 60s min-interval caps it to once a minute; reports 0 when idle. Triggered by the source broadcast events (item_progress, etc.) so it reaches the MQTT bridge; the session idles on dashcam_offline. - Web UI: live "Session avg X/s, ETA Y, Z this session" line. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 6 + tests/test_download_session.py | 166 ++++++++++++++++++++++++ tests/test_hub_session_stats.py | 190 ++++++++++++++++++++++++++++ tests/test_mqtt_hub_bridge.py | 13 ++ tests/test_mqtt_state_extraction.py | 42 ++++++ tests/test_mqtt_topology.py | 36 ++++++ tests/test_queue_pending_bytes.py | 38 ++++++ web/app.py | 9 +- web/services/download_session.py | 163 ++++++++++++++++++++++++ web/services/hub.py | 83 +++++++++++- web/services/mqtt_state.py | 24 ++++ web/services/mqtt_topology.py | 17 +++ web/services/queue.py | 15 +++ web/static/app.js | 34 ++++- web/static/index.html | 1 + web/static/styles.css | 7 + 16 files changed, 841 insertions(+), 3 deletions(-) create mode 100644 tests/test_download_session.py create mode 100644 tests/test_hub_session_stats.py create mode 100644 tests/test_queue_pending_bytes.py create mode 100644 web/services/download_session.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fd66d7d..c9f3bd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ - New `DISK_CRITICAL_PCT` setting (Snapshot field `disk_critical_pct`) configures the disk-pressure threshold above which sync goes into `error`. Must be `>= RETENTION_DISK_PCT`. +- The download manager now shows a session-wide moving-average download + speed and an estimated time to complete while a sync is running. +- New Home Assistant `download_speed` sensor (`data_rate`, MB/s) reports + the session moving average. To avoid flooding HA it publishes first at + ~30 s into a session then at most once per minute, and reports `0` when + idle. Enabled by default. ### Changed - Unified `sync_status` to four states: `downloading`, `waiting`, diff --git a/tests/test_download_session.py b/tests/test_download_session.py new file mode 100644 index 0000000..97bfd74 --- /dev/null +++ b/tests/test_download_session.py @@ -0,0 +1,166 @@ +"""Unit tests for the session download-speed + ETA tracker.""" +from __future__ import annotations + +from web.services.download_session import DownloadSession + + +class _Clock: + def __init__(self, t=0.0): + self.t = t + + def __call__(self): + return self.t + + +def _mk(remaining=0, *, clock=None, window_s=30.0): + c = clock or _Clock() + s = DownloadSession(lambda: remaining, monotonic=c, window_s=window_s) + return s, c + + +def test_inactive_until_started(): + s, _ = _mk() + assert s.active is False + assert s.avg_speed_bps is None + assert s.eta_seconds is None + assert s.snapshot()["active"] is False + + +def test_avg_speed_none_below_min_span(): + s, c = _mk() + s.note_started("a", 1000) + c.t = 0.5 + s.note_progress("a", 200, 1000) # one sample → no span + assert s.avg_speed_bps is None + c.t = 1.0 + s.note_progress("a", 400, 1000) # span 0.5 s < 2 s min + assert s.avg_speed_bps is None + + +def test_windowed_average_speed(): + s, c = _mk() + s.note_started("a", 10_000) + c.t = 1.0 + s.note_progress("a", 1000, 10_000) + c.t = 5.0 + s.note_progress("a", 5000, 10_000) + # (5000-1000)/(5-1) = 1000 B/s + assert s.avg_speed_bps == 1000.0 + + +def test_wire_bytes_monotonic_across_file_boundary(): + s, c = _mk() + s.note_started("a", 1000) + c.t = 1.0 + s.note_progress("a", 1000, 1000) + s.note_finished("a", 1000) + s.note_started("b", 2000) + c.t = 2.0 + s.note_progress("b", 500, 2000) + assert s.session_bytes == 1500 # 1000 (a) + 500 (b) + + +def test_retry_within_file_no_negative_delta(): + """download_file retries reset bytes_done WITHOUT a new item_started. + The wire-byte counter must clamp the backward jump to zero.""" + s, c = _mk() + s.note_started("a", 50_000_000) + c.t = 1.0 + s.note_progress("a", 30_000_000, 50_000_000) + # Connection drop → retry resumes from a small chunk, same file. + c.t = 2.0 + s.note_progress("a", 65_536, 50_000_000) + assert s.session_bytes == 30_000_000 # no decrease + c.t = 3.0 + s.note_progress("a", 131_072, 50_000_000) + assert s.session_bytes == 30_000_000 + 65_536 # climbs by the retry delta + assert (s.avg_speed_bps or 0) >= 0 + + +def test_eta_from_remaining_and_speed(): + s, c = _mk(remaining=9000) + s.note_started("a", 10_000) + c.t = 1.0 + s.note_progress("a", 1000, 10_000) + c.t = 5.0 + s.note_progress("a", 5000, 10_000) + # speed 1000 B/s; remaining = pending 9000 + current remainder 5000 = 14000 + assert s.avg_speed_bps == 1000.0 + assert s.eta_seconds == 14.0 + + +def test_eta_none_when_no_speed(): + s, c = _mk(remaining=9000) + s.note_started("a", 10_000) + c.t = 0.5 + s.note_progress("a", 1000, 10_000) + assert s.avg_speed_bps is None + assert s.eta_seconds is None + + +def test_refresh_remaining_picks_up_new_value(): + box = {"v": 1000} + c = _Clock() + s = DownloadSession(lambda: box["v"], monotonic=c, window_s=30.0) + s.note_started("a", 10_000) + c.t = 1.0 + s.note_progress("a", 1000, 10_000) + c.t = 5.0 + s.note_progress("a", 5000, 10_000) + assert s.eta_seconds == 6.0 # (1000 + 5000) / 1000 + box["v"] = 20_000 + s.refresh_remaining() + assert s.eta_seconds == 25.0 # (20000 + 5000) / 1000 + + +def test_note_idle_resets(): + s, c = _mk(remaining=9000) + s.note_started("a", 10_000) + c.t = 1.0 + s.note_progress("a", 1000, 10_000) + c.t = 5.0 + s.note_progress("a", 5000, 10_000) + s.note_idle() + assert s.active is False + assert s.session_bytes == 0 + assert s.avg_speed_bps is None + assert s.eta_seconds is None + assert s._remaining_pending == 0 + assert s.snapshot() == { + "active": False, "avg_speed_bps": None, "eta_seconds": None, + "session_bytes": 0, "elapsed_s": 0.0, + } + + +def test_old_samples_pruned_outside_window(): + s, c = _mk(window_s=10.0) + s.note_started("a", 100_000) + for t, b in [(1, 1000), (2, 2000), (12, 12000), (20, 30000)]: + c.t = float(t) + s.note_progress("a", b, 100_000) + # cutoff = 20 - 10 = 10; (1,*) and (2,*) drop while >2 samples remain. + assert len(s._samples) == 2 + assert s._samples[0][0] == 12.0 + assert s.avg_speed_bps == (30000 - 12000) / (20 - 12) + + +def test_progress_without_started_starts_session(): + """A stray item_progress with no prior item_started still begins a + session rather than being dropped.""" + s, c = _mk() + c.t = 1.0 + s.note_progress("a", 1000, 10_000) + assert s.active is True + assert s.session_bytes == 1000 + + +def test_lifespan_wires_download_session(tmp_config_dir, tmp_recordings_dir): + from fastapi.testclient import TestClient + from web import app as app_mod + from web import settings as settings_mod + settings_mod.reset_for_tests() + app = app_mod.create_app() + with TestClient(app): + assert isinstance(app.state.download_session, DownloadSession) + # The Hub holds the same tracker instance it feeds. + assert app.state.hub._session is app.state.download_session diff --git a/tests/test_hub_session_stats.py b/tests/test_hub_session_stats.py new file mode 100644 index 0000000..2111646 --- /dev/null +++ b/tests/test_hub_session_stats.py @@ -0,0 +1,190 @@ +"""Hub feeds the DownloadSession tracker and emits session_stats.""" +from __future__ import annotations + +from web.services.download_session import DownloadSession +from web.services.hub import Hub + + +class _Clock: + def __init__(self, t=0.0): + self.t = t + + def __call__(self): + return self.t + + +class _RecordingWS: + def __init__(self): + self.sent = [] + + async def accept(self): + pass + + async def send_json(self, payload): + self.sent.append(payload) + + +def _hub_with_session(remaining=1000, clock=None): + c = clock or _Clock() + sess = DownloadSession(lambda: remaining, monotonic=c, window_s=30.0) + return Hub(session=sess), c + + +async def test_initial_last_state_has_idle_session(): + hub = Hub() + assert "session" in hub.last_state + assert hub.last_state["session"]["active"] is False + + +async def test_progress_drives_session_stats_event(): + hub, c = _hub_with_session() + ws = _RecordingWS() + await hub.connect(ws) + ws.sent.clear() + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + c.t = 6.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 6000, "total": 10_000}) + stats = [e for e in ws.sent if e.get("type") == "session_stats"] + assert stats, "expected at least one session_stats event" + assert stats[-1]["active"] is True + assert stats[-1]["avg_speed_bps"] is not None + assert hub.last_state["session"]["active"] is True + + +async def test_sync_done_emits_idle_session_stats(): + hub, c = _hub_with_session() + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + ws.sent.clear() + await hub.broadcast({"type": "sync_done", "ok": True}) + assert hub.last_state["session"]["active"] is False + idle = [e for e in ws.sent if e.get("type") == "session_stats"] + assert idle and idle[-1]["active"] is False + + +async def test_running_sync_state_does_not_reset_session(): + """sync_state with running=True fires every time the worker picks an + item — it must NOT idle an in-flight session.""" + hub, c = _hub_with_session() + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + await hub.broadcast({"type": "sync_state", "running": True, + "paused": False}) + assert hub.last_state["session"]["active"] is True + + +async def test_paused_sync_state_resets_session(): + hub, c = _hub_with_session() + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + await hub.broadcast({"type": "sync_state", "running": True, + "paused": True}) + assert hub.last_state["session"]["active"] is False + + +async def test_session_stats_deduped_when_rounded_view_unchanged(): + hub, c = _hub_with_session() + ws = _RecordingWS() + await hub.connect(ws) + # Get into a steady active state with a computed speed. + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + c.t = 6.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 6000, "total": 10_000}) + ws.sent.clear() + # An unrelated event that doesn't advance the clock or bytes: the + # rounded session view is identical → no duplicate session_stats. + await hub.broadcast({"type": "dashcam_online"}) + dup = [e for e in ws.sent if e.get("type") == "session_stats"] + assert dup == [] + + +async def test_snapshot_carries_session(): + hub, c = _hub_with_session() + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + ws = _RecordingWS() + await hub.connect(ws) + snap = ws.sent[0] + assert snap["type"] == "snapshot" + assert snap["state"]["session"]["active"] is True + + +async def test_no_session_tracker_is_safe(): + """Hub with session=None must not emit session_stats or raise.""" + hub = Hub() # no session + ws = _RecordingWS() + await hub.connect(ws) + ws.sent.clear() + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + assert not any(e.get("type") == "session_stats" for e in ws.sent) + + +async def test_active_session_emits_heartbeat_on_elapsed_change(): + """Even with steady speed/eta, advancing the clock during an active + session must still emit a session_stats (the ~1/s heartbeat that keeps + MQTT triggered).""" + c = _Clock() + # Constant remaining + linear progress → speed stays flat. + hub = Hub(session=DownloadSession(lambda: 1000, monotonic=c, window_s=30.0)) + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 1_000_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 1_000_000}) + c.t = 6.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 6000, "total": 1_000_000}) + ws.sent.clear() + # Advance the clock by ~1s; broadcast an unrelated event. elapsed + # changed → heartbeat session_stats expected. + c.t = 7.0 + await hub.broadcast({"type": "dashcam_online"}) + beats = [e for e in ws.sent if e.get("type") == "session_stats"] + assert beats, "expected a heartbeat session_stats after elapsed advanced" + + +async def test_dashcam_offline_resets_session(): + """A mid-download camera drop must idle the session so the UI line and + HA sensor don't freeze on a stale speed.""" + hub, c = _hub_with_session() + ws = _RecordingWS() + await hub.connect(ws) + await hub.broadcast({"type": "item_started", "filename": "a.mp4", + "total": 10_000}) + c.t = 3.0 + await hub.broadcast({"type": "item_progress", "filename": "a.mp4", + "bytes": 3000, "total": 10_000}) + assert hub.last_state["session"]["active"] is True + await hub.broadcast({"type": "dashcam_offline"}) + assert hub.last_state["session"]["active"] is False diff --git a/tests/test_mqtt_hub_bridge.py b/tests/test_mqtt_hub_bridge.py index a57651e..1c7c62c 100644 --- a/tests/test_mqtt_hub_bridge.py +++ b/tests/test_mqtt_hub_bridge.py @@ -19,3 +19,16 @@ def test_clip_indexed_affects_archive_entities(): def test_unknown_event_yields_empty(): from web.services.mqtt import entities_affected_by assert list(entities_affected_by("this_event_does_not_exist")) == [] + + +def test_download_speed_triggered_by_broadcast_events(): + """Regression: download_speed must trigger on source events that flow + through Hub.broadcast (e.g. item_progress), NOT the session_stats + follow-up — that is emitted via a direct send and never reaches the + MQTT bridge, so an entity keyed on it would never re-publish.""" + from web.services.mqtt import entities_affected_by + assert "download_speed" in { + e.object_id for e in entities_affected_by("item_progress") + } + # session_stats is a follow-up event; no entity should depend on it. + assert list(entities_affected_by("session_stats")) == [] diff --git a/tests/test_mqtt_state_extraction.py b/tests/test_mqtt_state_extraction.py index 28112d9..78010a2 100644 --- a/tests/test_mqtt_state_extraction.py +++ b/tests/test_mqtt_state_extraction.py @@ -311,3 +311,45 @@ def test_state_disk_used_missing_path_returns_none(tmp_path): assert out is None +# ---- download speed (session moving average) + +def test_state_download_speed_none_before_30s(): + from web.services.mqtt_state import state_download_speed + hub = _hub_with_state({"session": { + "active": True, "elapsed_s": 10.0, + "avg_speed_bps": float(5 * 1024 * 1024), + }}) + assert state_download_speed(hub, None, _stub_snapshot()) is None + + +def test_state_download_speed_value_after_30s(): + from web.services.mqtt_state import state_download_speed + hub = _hub_with_state({"session": { + "active": True, "elapsed_s": 35.0, + "avg_speed_bps": float(2 * 1024 * 1024), + }}) + assert state_download_speed(hub, None, _stub_snapshot()) == "2.0" + + +def test_state_download_speed_zero_when_idle(): + from web.services.mqtt_state import state_download_speed + hub = _hub_with_state({"session": { + "active": False, "elapsed_s": 0.0, "avg_speed_bps": None, + }}) + assert state_download_speed(hub, None, _stub_snapshot()) == "0" + + +def test_state_download_speed_zero_when_no_session_key(): + from web.services.mqtt_state import state_download_speed + hub = _hub_with_state({}) + assert state_download_speed(hub, None, _stub_snapshot()) == "0" + + +def test_state_download_speed_none_when_avg_unavailable(): + from web.services.mqtt_state import state_download_speed + hub = _hub_with_state({"session": { + "active": True, "elapsed_s": 35.0, "avg_speed_bps": None, + }}) + assert state_download_speed(hub, None, _stub_snapshot()) is None + + diff --git a/tests/test_mqtt_topology.py b/tests/test_mqtt_topology.py index 8811c05..d71950c 100644 --- a/tests/test_mqtt_topology.py +++ b/tests/test_mqtt_topology.py @@ -136,3 +136,39 @@ def test_state_entities_default_to_global_qos(): by_id = {e.object_id: e for e in TOPOLOGY} assert by_id["sync_status"].qos is None assert by_id["dashcam"].qos is None + + +def test_download_speed_entity_present(): + from web.services.mqtt_topology import TOPOLOGY + assert "download_speed" in {e.object_id for e in TOPOLOGY} + + +def test_download_speed_entity_config(): + from web.services.mqtt_topology import TOPOLOGY + from web.services.mqtt_state import state_download_speed + e = next(x for x in TOPOLOGY if x.object_id == "download_speed") + assert e.component == "sensor" + assert e.device_class == "data_rate" + assert e.unit_of_measurement == "MB/s" + assert e.state_class == "measurement" + assert e.enabled_by_default is True + assert e.min_publish_interval_s == 60.0 + assert e.qos == 0 + assert e.affected_by_hub_events == ( + "item_progress", "item_started", "item_finished", + "sync_done", "sync_state", "dashcam_offline", + ) + assert e.state_fn is state_download_speed + + +def test_download_speed_discovery_payload(): + from web.services.mqtt_topology import TOPOLOGY, build_discovery_payload + e = next(x for x in TOPOLOGY if x.object_id == "download_speed") + cfg = {"discovery_prefix": "homeassistant", "node_id": "viofosync", + "version": "1", "configuration_url": ""} + p = build_discovery_payload(e, cfg) + assert p["device_class"] == "data_rate" + assert p["unit_of_measurement"] == "MB/s" + assert p["state_class"] == "measurement" + assert p["enabled_by_default"] is True + assert p["state_topic"] == "viofosync/download_speed/state" diff --git a/tests/test_queue_pending_bytes.py b/tests/test_queue_pending_bytes.py new file mode 100644 index 0000000..e9a4ac6 --- /dev/null +++ b/tests/test_queue_pending_bytes.py @@ -0,0 +1,38 @@ +"""pending_bytes() sums remote_size across pending rows only.""" +from __future__ import annotations + +import time + + +def _db_with_rows(tmp_path, rows): + """rows: list of (filename, state, remote_size).""" + from web.db import Database + db = Database(str(tmp_path / "v.db")) + now = int(time.time()) + with db.write() as c: + for (filename, state, size) in rows: + c.execute( + "INSERT INTO download_queue " + "(filename, source_dir, state, remote_size, enqueued_at) " + "VALUES (?, ?, ?, ?, ?)", + (filename, "/DCIM/Movie", state, size, now), + ) + return db + + +def test_pending_bytes_sums_only_pending(tmp_path): + from web.services.queue import pending_bytes + db = _db_with_rows(tmp_path, [ + ("a.MP4", "pending", 100), + ("b.MP4", "pending", 200), + ("c.MP4", "done", 999), + ("d.MP4", "failed", 50), + ("e.MP4", "downloading", 77), + ]) + assert pending_bytes(db) == 300 + + +def test_pending_bytes_zero_when_empty(tmp_path): + from web.services.queue import pending_bytes + db = _db_with_rows(tmp_path, []) + assert pending_bytes(db) == 0 diff --git a/web/app.py b/web/app.py index d646bf1..21d84ec 100644 --- a/web/app.py +++ b/web/app.py @@ -40,6 +40,7 @@ probe_encoders, ) from .services.geocode import GeocodeService +from .services.download_session import DownloadSession from .services.hub import Hub from .services.mqtt import MqttService from .services.sync_worker import SyncWorker @@ -142,7 +143,13 @@ async def _background_retention() -> None: app.state.retention_task = asyncio.create_task(_background_retention()) - app.state.hub = Hub(settings_provider=provider) + app.state.download_session = DownloadSession( + remaining_bytes_provider=lambda: _q_mod.pending_bytes(app.state.db), + ) + app.state.hub = Hub( + settings_provider=provider, + session=app.state.download_session, + ) # Compute initial sync_status so the very first WebSocket snapshot # and the first MQTT publish carry the right value without waiting # for the sync worker's first cycle. diff --git a/web/services/download_session.py b/web/services/download_session.py new file mode 100644 index 0000000..dd17e7e --- /dev/null +++ b/web/services/download_session.py @@ -0,0 +1,163 @@ +"""Session-level download throughput + ETA tracking. + +A :class:`DownloadSession` accumulates progress events across the files +of one download run and exposes a windowed moving-average speed plus an +ETA. All math lives here — no broker, DB, or socket I/O — so it is +unit-testable with an injected clock and a stubbed remaining-bytes +provider. + +It is fed exclusively from ``Hub.broadcast`` on the event loop, so no +locking is required. +""" +from __future__ import annotations + +import time +from collections import deque +from typing import Callable, Deque, Optional, Tuple + + +class DownloadSession: + # Below this many seconds of sample span the windowed speed is too + # noisy to report. + _MIN_SPAN_S = 2.0 + + def __init__( + self, + remaining_bytes_provider: Callable[[], int], + *, + monotonic: Callable[[], float] = time.monotonic, + window_s: float = 30.0, + ) -> None: + self._remaining_provider = remaining_bytes_provider + self._mono = monotonic + self._window_s = window_s + self._active = False + self._started_at: Optional[float] = None + self._wire_bytes = 0 + self._samples: Deque[Tuple[float, int]] = deque() + self._cur_file: Optional[str] = None + self._cur_file_bytes = 0 + self._cur_total: Optional[int] = None + self._remaining_pending = 0 + + # ---- event feeds (called on the loop) ---- + + def note_started(self, filename: str, total: Optional[int]) -> None: + if not self._active: + self._active = True + self._started_at = self._mono() + self._wire_bytes = 0 + self._samples.clear() + self._cur_file = filename + self._cur_file_bytes = 0 + self._cur_total = total + self._refresh_remaining() + + def note_progress( + self, filename: str, bytes_done: int, total: Optional[int], + ) -> None: + if not self._active: + self.note_started(filename, total) + if filename == self._cur_file: + delta = bytes_done - self._cur_file_bytes + else: + # A file we never saw an item_started for. + self._cur_file = filename + delta = bytes_done + if delta < 0: + # A retry reset bytes_done within the same file — never let the + # monotonic counter go backwards. + delta = 0 + self._wire_bytes += delta + self._cur_file_bytes = bytes_done + self._cur_total = total + now = self._mono() + self._samples.append((now, self._wire_bytes)) + self._prune(now) + + def note_finished( + self, filename: str, bytes_written: Optional[int], + ) -> None: + # Progress ticks already accounted for the bytes; just clear the + # per-file cursor so the next file starts fresh. + self._cur_file = None + self._cur_file_bytes = 0 + self._cur_total = None + self._refresh_remaining() + + def note_idle(self) -> None: + self._active = False + self._started_at = None + self._wire_bytes = 0 + self._samples.clear() + self._cur_file = None + self._cur_file_bytes = 0 + self._cur_total = None + self._remaining_pending = 0 + + def refresh_remaining(self) -> None: + self._refresh_remaining() + + # ---- derived values ---- + + @property + def active(self) -> bool: + return self._active + + @property + def elapsed_s(self) -> float: + if not self._active or self._started_at is None: + return 0.0 + return max(0.0, self._mono() - self._started_at) + + @property + def session_bytes(self) -> int: + return self._wire_bytes + + @property + def avg_speed_bps(self) -> Optional[float]: + if len(self._samples) < 2: + return None + t0, b0 = self._samples[0] + t1, b1 = self._samples[-1] + span = t1 - t0 + if span < self._MIN_SPAN_S: + return None + return max(0.0, (b1 - b0) / span) + + @property + def eta_seconds(self) -> Optional[float]: + speed = self.avg_speed_bps + if not speed: # None or 0 + return None + remaining = self._remaining_pending + if self._cur_total: + remaining += max(0, self._cur_total - self._cur_file_bytes) + if remaining <= 0: + return 0.0 + return remaining / speed + + def snapshot(self) -> dict: + return { + "active": self._active, + "avg_speed_bps": self.avg_speed_bps, + "eta_seconds": self.eta_seconds, + "session_bytes": self._wire_bytes, + "elapsed_s": self.elapsed_s, + } + + # ---- internals ---- + + def _prune(self, now: float) -> None: + cutoff = now - self._window_s + # Always keep at least 2 samples so a steady transfer still has a + # span to measure; drop older ones once a newer in-window sample + # exists. + while len(self._samples) > 2 and self._samples[0][0] < cutoff: + self._samples.popleft() + + def _refresh_remaining(self) -> None: + try: + self._remaining_pending = int(self._remaining_provider() or 0) + except Exception: + self._remaining_pending = 0 diff --git a/web/services/hub.py b/web/services/hub.py index e81bccb..dde6e5d 100644 --- a/web/services/hub.py +++ b/web/services/hub.py @@ -28,16 +28,25 @@ class Hub: - def __init__(self, settings_provider: Any = None) -> None: + def __init__(self, settings_provider: Any = None, session: Any = None) -> None: self._clients: Set[WebSocket] = set() self._lock = asyncio.Lock() self._settings_provider = settings_provider + self._session = session + # Last broadcast session_stats key, for deduping the follow-up. + self._last_session_key: Any = None # Retain the last snapshot of major state so a newly- # connected client sees the current situation without # waiting for the next event. self.last_state: Dict[str, Any] = { "dashcam_online": None, "current_item": None, + # Session-wide download stats (see download_session.py). Always + # present so the WS snapshot and MQTT state_fn never KeyError. + "session": { + "active": False, "avg_speed_bps": None, "eta_seconds": None, + "session_bytes": 0, "elapsed_s": 0.0, + }, # Stateful diagnostics consumed by compute_sync_status(): "sync_error": None, "disk_pct": None, @@ -121,6 +130,9 @@ async def broadcast(self, event: Dict[str, Any]) -> None: if isinstance(pct, (int, float)): self.last_state["disk_pct"] = float(pct) + # Feed the session tracker (reads the event; mutates the tracker). + self._feed_session(t, event) + dead: list = [] async with self._lock: clients = list(self._clients) @@ -131,6 +143,7 @@ async def broadcast(self, event: Dict[str, Any]) -> None: dead.append(ws) # Recompute the unified status after the state mutation above. await self._maybe_emit_sync_status(dead) + await self._maybe_emit_session_stats(dead) if dead: async with self._lock: for ws in dead: @@ -169,6 +182,74 @@ async def _maybe_emit_sync_status(self, dead: list) -> None: except Exception: dead.append(ws) + def _feed_session(self, t: str, event: Dict[str, Any]) -> None: + """Translate the relevant Hub events into DownloadSession calls. + Runs on the event loop, so the tracker needs no locking.""" + s = self._session + if s is None: + return + try: + if t == "item_started": + s.note_started(event.get("filename"), event.get("total")) + elif t == "item_progress": + s.note_progress( + event.get("filename"), event.get("bytes"), + event.get("total"), + ) + elif t == "item_finished": + s.note_finished(event.get("filename"), event.get("bytes")) + elif t == "sync_done": + s.note_idle() + elif t == "dashcam_offline": + s.note_idle() + elif t == "sync_state": + # running=True fires on every item pick — only idle the + # session on the stopped/paused variant. + if not event.get("running") or event.get("paused"): + s.note_idle() + elif t in ("queue_reconciled", "queue_changed"): + s.refresh_remaining() + except Exception: + log.exception("session tracker feed failed for %s", t) + + async def _maybe_emit_session_stats(self, dead: list) -> None: + """Store the latest session snapshot in last_state and broadcast a + session_stats follow-up when the rounded view changes. Mirrors + _maybe_emit_sync_status; ``dead`` accumulates disconnected clients.""" + s = self._session + if s is None: + return + try: + snap = s.snapshot() + except Exception: + log.exception("session snapshot failed; skipping follow-up") + return + self.last_state["session"] = snap + # Dedupe on a rounded view so sub-noise jitter doesn't broadcast. + speed = snap.get("avg_speed_bps") + eta = snap.get("eta_seconds") + # Include whole-second elapsed so an active session emits a ~1/s + # heartbeat even when speed/eta are stable — this keeps the MQTT + # publisher triggered and the UI byte counter fresh. Idle sessions + # hold elapsed at 0.0, so they stay silent. + key = ( + snap.get("active"), + None if speed is None else round(speed / (1024 * 1024), 1), + None if eta is None else round(eta), + round(snap.get("elapsed_s") or 0.0), + ) + if key == self._last_session_key: + return + self._last_session_key = key + event = {"type": "session_stats", **snap} + async with self._lock: + clients = list(self._clients) + for ws in clients: + try: + await ws.send_json(event) + except Exception: + dead.append(ws) + def schedule_broadcast( self, loop: asyncio.AbstractEventLoop, diff --git a/web/services/mqtt_state.py b/web/services/mqtt_state.py index fb5f19a..53f05fc 100644 --- a/web/services/mqtt_state.py +++ b/web/services/mqtt_state.py @@ -113,6 +113,30 @@ def state_current_progress(hub, db, snapshot) -> Optional[str]: return f"{pct}" +# Suppress publishing the session speed until the window has filled and +# the average has stabilised. +SPEED_PUBLISH_DELAY_S = 30.0 + + +def state_download_speed(hub, db, snapshot) -> Optional[str]: + """Session moving-average download speed in MB/s. + + Returns ``None`` (no publish) for the first ``SPEED_PUBLISH_DELAY_S`` + of a session or before the average is computable; ``"0"`` when idle. + Combined with the entity's 60 s ``min_publish_interval_s`` this yields + a first publish at ~30 s then at most once per 60 s. + """ + sess = hub.last_state.get("session") or {} + if not sess.get("active"): + return "0" + if (sess.get("elapsed_s") or 0) < SPEED_PUBLISH_DELAY_S: + return None + bps = sess.get("avg_speed_bps") + if bps is None: + return None + return f"{bps / (1024 * 1024):.1f}" + + # ---- disk def state_disk_used(hub, db, snapshot) -> Optional[str]: diff --git a/web/services/mqtt_topology.py b/web/services/mqtt_topology.py index 0202fee..84bea5e 100644 --- a/web/services/mqtt_topology.py +++ b/web/services/mqtt_topology.py @@ -265,6 +265,23 @@ def build_discovery_payload(entity: EntityDef, cfg: dict) -> dict: "item_finished"), qos=0, ), + EntityDef( + object_id="download_speed", + component="sensor", + name="Download speed", + icon="mdi:speedometer", + device_class="data_rate", + state_class="measurement", + unit_of_measurement="MB/s", + enabled_by_default=True, + min_publish_interval_s=60.0, + state_fn=_st.state_download_speed, + command_handler=None, + affected_by_hub_events=("item_progress", "item_started", + "item_finished", "sync_done", "sync_state", + "dashcam_offline"), + qos=0, + ), # --- Disk / sync history --- EntityDef( diff --git a/web/services/queue.py b/web/services/queue.py index 5dded0a..3e42957 100644 --- a/web/services/queue.py +++ b/web/services/queue.py @@ -565,6 +565,21 @@ def list_day_items( return [dict(r) for r in rows] +def pending_bytes(db: Database) -> int: + """Total ``remote_size`` across all rows in state ``pending``. + + Feeds the session ETA. HTML-listing rows are MB-rounded, which is + fine for an estimate; rows are corrected to byte-exact sizes after + each download. + """ + with db.conn() as c: + row = c.execute( + "SELECT COALESCE(SUM(remote_size), 0) AS n " + "FROM download_queue WHERE state='pending'" + ).fetchone() + return int(row["n"]) + + def prioritize_recent_hours(db: Database, hours: float) -> int: """Bump all pending items recorded in the last ``hours`` hours to the top of the queue. Returns the count updated.""" diff --git a/web/static/app.js b/web/static/app.js index 37f040a..eeea657 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -574,6 +574,13 @@ function fmtDuration(seconds) { return rem ? `${h}h ${rem}m` : `${h}h`; } +// ETA wants sub-minute precision; fmtDuration rounds those to "0 min". +function fmtEta(seconds) { + if (seconds == null) return "—"; + if (seconds < 60) return `${Math.round(seconds)}s`; + return fmtDuration(seconds); +} + function renderStopCard(stop, clips, idx) { const startT = new Date(stop.start_time).toLocaleTimeString(); const endT = new Date(stop.end_time).toLocaleTimeString(); @@ -1659,7 +1666,8 @@ function appendLog(ev) { // Per-chunk progress is already shown live in the progress bar // above (downloads) or the Export jobs table (exports); mirroring // it in the log buries everything else. - if (ev.type === "item_progress" || ev.type === "export_progress") return; + if (ev.type === "item_progress" || ev.type === "export_progress" + || ev.type === "session_stats") return; const line = document.createElement("div"); line.className = "log-line"; const ts = new Date().toLocaleTimeString(); @@ -1721,6 +1729,7 @@ function handleEvent(ev) { applyStatus(ev.state.sync_status, ev.state.sync_status_reason); } if (ev.state.current_item) updateCurrent(ev.state.current_item); + if (ev.state.session) updateSessionStats(ev.state.session); if (ev.state.sync_state) { state.syncRunning = ev.state.sync_state.running; state.syncPaused = ev.state.sync_state.paused; @@ -1751,6 +1760,9 @@ function handleEvent(ev) { state.currentFilename = null; refreshQueueIfVisible(); break; + case "session_stats": + updateSessionStats(ev); + break; case "queue_reconciled": case "sync_done": refreshQueueIfVisible(); @@ -1817,6 +1829,26 @@ function updateCurrent(info) { state.currentFilename = info.filename; } +function updateSessionStats(s) { + const el = document.getElementById("session-stats"); + if (!el) return; + if (!s || !s.active) { + el.hidden = true; + el.textContent = ""; + return; + } + const parts = []; + if (s.avg_speed_bps != null) { + parts.push(`avg ${fmtBytes(s.avg_speed_bps)}/s`); + } + if (s.eta_seconds != null) { + parts.push(`ETA ${fmtEta(s.eta_seconds)}`); + } + parts.push(`${fmtBytes(s.session_bytes)} this session`); + el.textContent = "Session · " + parts.join(" · "); + el.hidden = false; +} + // ---------- Settings ---------- // // Section renderers paint into #settings-pane based on the hash diff --git a/web/static/index.html b/web/static/index.html index ac91fab..9175ed3 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -110,6 +110,7 @@

Archive

Download manager

+
From bddefb101d0bc6c6e3a20aaa44b469007e51cf89 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 4 Jun 2026 21:27:57 +0100 Subject: [PATCH 13/21] feat(downloads): group the download list by hour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Break each day's download list into a collapsible hour tier. Opening a day now shows a list of hours (newest first), each with its clip count, size, and state pills (downloading/pending/done/failed/gone); drilling into an hour reveals the file table. A tri-state select-all cascade spans day -> hour -> file, all backed by the existing flat filename selection set. This keeps a busy day (hundreds of clips) scannable instead of painting one giant table on expand. Pure client-side change — rendering and selection only; the backend, API, and SQL are untouched. Also ignores the local .superpowers/ brainstorm scratch dir. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 3 + web/static/app.js | 196 ++++++++++++++++++++++++++++++++++++++++-- web/static/styles.css | 63 +++++++++++--- 3 files changed, 242 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 68483c5..614ce67 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ config.json .venv/ docs/* CLAUDE.md + +# Superpowers brainstorming companion (local mockups) +.superpowers/ diff --git a/web/static/app.js b/web/static/app.js index 22f7493..3a374e5 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -15,6 +15,7 @@ const state = { queueDays: [], // list of day summaries from /api/queue/days queueDayItems: {}, // { 'YYYY-MM-DD': [items] } for expanded days queueExpanded: new Set(), + queueHoursExpanded: new Set(), // keys: "YYYY-MM-DD HH" (HH may be "??") queueSelected: new Set(),// filenames ticked filters: { driving: true, parking: true, ro: true }, showMaps: localStorage.getItem("vfs.showMaps") !== "0", @@ -1362,6 +1363,50 @@ function renderPagination(total) { // ---------- Downloads ---------- +// Two-digit hour ("00".."23") for a queue item, derived from the +// dashcam filename (YYYY_MMDD_HHMMSS_…). Filename-derived (not +// recorded_at) to stay timezone-stable and consistent with the +// server's day grouping (_day_expr in services/queue.py). Names that +// don't match bucket under "??" so they stay visible and sort last. +function hourKeyForItem(it) { + const m = /^\d{4}_\d{4}_(\d{2})/.exec(it.filename || ""); + return m ? m[1] : "??"; +} + +// Bucket a day's items by hour. Returns [{ hour, items }] with hours +// newest-first and any "??" bucket last. +function groupItemsByHour(items) { + const buckets = new Map(); + for (const it of items) { + const hh = hourKeyForItem(it); + if (!buckets.has(hh)) buckets.set(hh, []); + buckets.get(hh).push(it); + } + const keys = [...buckets.keys()].sort((a, b) => { + if (a === "??") return 1; + if (b === "??") return -1; + return Number(b) - Number(a); + }); + return keys.map((hh) => ({ hour: hh, items: buckets.get(hh) })); +} + +// Per-hour counts + summed bytes for the hour row, computed client-side +// from the bucket. State counts use bare keys (pending/downloading/done/ +// failed/gone) and are consumed only by renderQueueHour — not the +// server-derived day summaries (which use *_count) — so they deliberately +// diverge from that naming. +function hourSummary(items) { + const s = { + clip_count: items.length, total_bytes: 0, + pending: 0, downloading: 0, done: 0, failed: 0, gone: 0, + }; + for (const it of items) { + s.total_bytes += it.remote_size || 0; + if (Object.prototype.hasOwnProperty.call(s, it.state)) s[it.state]++; + } + return s; +} + function queueKindParams(q) { q.set("driving", state.queueKinds.driving ? "true" : "false"); q.set("parking", state.queueKinds.parking ? "true" : "false"); @@ -1380,6 +1425,11 @@ async function loadQueue() { for (const d of Object.keys(state.queueDayItems)) { if (!liveDays.has(d)) delete state.queueDayItems[d]; } + for (const key of [...state.queueHoursExpanded]) { + if (!liveDays.has(key.split(" ")[0])) { + state.queueHoursExpanded.delete(key); + } + } // Refresh items for any expanded days so live counts stay in sync. await Promise.all( [...state.queueExpanded] @@ -1507,20 +1557,105 @@ function renderQueueDayCard(d) { if (expanded && state.queueDayItems[d.day]) { const body = el.querySelector(".queue-day-body"); - body.appendChild(renderDayItemsTable(d.day, state.queueDayItems[d.day])); + body.appendChild(renderDayHours(d.day, state.queueDayItems[d.day])); } return el; } -function renderDayItemsTable(day, items) { +function renderDayHours(day, items) { const wrap = document.createElement("div"); + wrap.className = "queue-hours"; if (!items.length) { wrap.innerHTML = `

No files match this filter.

`; return wrap; } + for (const group of groupItemsByHour(items)) { + wrap.appendChild(renderQueueHour(day, group.hour, group.items)); + } + return wrap; +} + +function renderQueueHour(day, hh, items) { + const el = document.createElement("div"); + el.className = "queue-hour"; + el.dataset.hour = hh; + const key = `${day} ${hh}`; + const expanded = state.queueHoursExpanded.has(key); + const s = hourSummary(items); + const checkState = hourCheckState(day, hh); + const hasPending = s.pending > 0; + const label = hh === "??" ? "Unknown time" : `${hh}:00–${hh}:59`; + + const pieces = []; + if (s.downloading) pieces.push(`${s.downloading} downloading`); + if (s.pending) pieces.push(`${s.pending} pending`); + if (s.done) pieces.push(`${s.done} done`); + if (s.failed) pieces.push(`${s.failed} failed`); + if (s.gone) pieces.push(`${s.gone} gone`); + + el.innerHTML = ` +
+ ${expanded ? "▾" : "▸"} + + ${label} + ${s.clip_count} clips · ${fmtMB(s.total_bytes)} +
${pieces.join("")}
+
+
+ `; + + const checkbox = el.querySelector(".qh-check"); + if (checkState === "indeterminate") checkbox.indeterminate = true; + checkbox.addEventListener("click", (e) => e.stopPropagation()); + checkbox.addEventListener("change", (e) => { + // Recompute live: this row updates surgically (no full re-render), + // so a captured checkState would go stale across repeated clicks and + // break deselect on an hour that began indeterminate. + const live = hourCheckState(day, hh); + const shouldSelect = e.target.checked || live === "indeterminate"; + toggleHourSelection(day, hh, shouldSelect); + // Reflect onto any rendered file rows + recompute the headers, + // surgically — no full re-render, so scroll is preserved. + el.querySelectorAll(".queue-hour-body .qi-check").forEach((cb) => { + cb.checked = state.queueSelected.has(cb.value); + }); + updateHourHeaderCheckbox(day, hh); + updateDayHeaderCheckbox(day); + renderQueueMeta(); + }); + + const header = el.querySelector(".queue-hour-header"); + header.addEventListener("click", (e) => { + if (e.target.closest(".qh-check")) return; + const body = el.querySelector(".queue-hour-body"); + const caret = el.querySelector(".caret"); + if (state.queueHoursExpanded.has(key)) { + state.queueHoursExpanded.delete(key); + body.hidden = true; + body.innerHTML = ""; + caret.textContent = "▸"; + } else { + state.queueHoursExpanded.add(key); + body.appendChild(renderHourBody(day, hh, items)); + body.hidden = false; + caret.textContent = "▾"; + } + }); + + if (expanded) { + el.querySelector(".queue-hour-body") + .appendChild(renderHourBody(day, hh, items)); + } + return el; +} + +function renderHourBody(day, hh, items) { + const wrap = document.createElement("div"); const table = document.createElement("table"); table.className = "queue-items"; table.innerHTML = ` @@ -1539,8 +1674,7 @@ function renderDayItemsTable(day, items) { const tbody = table.querySelector("tbody"); for (const it of items) { const tr = document.createElement("tr"); - const size = it.remote_size - ? fmtMB(it.remote_size) : "—"; + const size = it.remote_size ? fmtMB(it.remote_size) : "—"; const ts = it.recorded_at ? new Date(it.recorded_at * 1000).toLocaleTimeString() : "—"; const pos = it.queue_position === 0 ? "▶" : @@ -1567,9 +1701,11 @@ function renderDayItemsTable(day, items) { if (!cb) return; if (cb.checked) state.queueSelected.add(cb.value); else state.queueSelected.delete(cb.value); - // Update just this day's header checkbox without a full re-render, - // so the user's scroll position isn't lost. + // Update this file's hour checkbox and the day checkbox in place, + // preserving scroll position (no re-render). + updateHourHeaderCheckbox(day, hh); updateDayHeaderCheckbox(day); + renderQueueMeta(); }); wrap.appendChild(table); return wrap; @@ -1618,6 +1754,54 @@ function updateDayHeaderCheckbox(day) { cb.checked = st === "checked"; } +// ---- Hour-level selection (twins of the day helpers above) ---- + +function itemsInHour(day, hh) { + const items = state.queueDayItems[day] || []; + return items.filter((it) => hourKeyForItem(it) === hh); +} + +function hourPendingCount(day, hh) { + let n = 0; + for (const it of itemsInHour(day, hh)) if (it.state === "pending") n++; + return n; +} + +function countSelectedInHour(day, hh) { + let n = 0; + for (const it of itemsInHour(day, hh)) { + if (it.state === "pending" && state.queueSelected.has(it.filename)) n++; + } + return n; +} + +function hourCheckState(day, hh) { + const pending = hourPendingCount(day, hh); + if (!pending) return "unchecked"; + const sel = countSelectedInHour(day, hh); + if (sel === 0) return "unchecked"; + if (sel >= pending) return "checked"; + return "indeterminate"; +} + +function toggleHourSelection(day, hh, select) { + for (const it of itemsInHour(day, hh)) { + if (it.state !== "pending") continue; + if (select) state.queueSelected.add(it.filename); + else state.queueSelected.delete(it.filename); + } +} + +function updateHourHeaderCheckbox(day, hh) { + const card = document.querySelector(`.queue-day[data-day="${day}"]`); + if (!card) return; + const cb = card.querySelector(`.qh-check[data-hour="${hh}"]`); + if (!cb) return; + const st = hourCheckState(day, hh); + cb.indeterminate = st === "indeterminate"; + cb.checked = st === "checked"; +} + function wireKindCheckbox(id, key) { document.getElementById(id).addEventListener("change", (e) => { state.queueKinds[key] = e.target.checked; diff --git a/web/static/styles.css b/web/static/styles.css index 42aed6e..78d3c9f 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -721,7 +721,7 @@ main { letter-spacing: -0.005em; } .queue-day-header .meta { flex: 0 0 auto; } -.queue-day-header .state-breakdown { +.state-breakdown { flex: 1; display: flex; gap: 6px; @@ -729,7 +729,7 @@ main { justify-content: flex-end; font-variant-numeric: tabular-nums; } -.queue-day-header .state-breakdown span { +.state-breakdown span { display: inline-flex; align-items: center; gap: 6px; @@ -743,7 +743,7 @@ main { color: var(--muted); white-space: nowrap; } -.queue-day-header .state-breakdown span::before { +.state-breakdown span::before { content: ""; width: 6px; height: 6px; @@ -751,38 +751,38 @@ main { flex: 0 0 auto; background: var(--muted); } -.queue-day-header .state-breakdown .state-pending { +.state-breakdown .state-pending { color: var(--muted); } -.queue-day-header .state-breakdown .state-pending::before { +.state-breakdown .state-pending::before { background: var(--muted); } -.queue-day-header .state-breakdown .state-downloading { +.state-breakdown .state-downloading { color: var(--accent); border-color: var(--accent-35); background: var(--accent-08); } -.queue-day-header .state-breakdown .state-downloading::before { +.state-breakdown .state-downloading::before { background: var(--accent); } -.queue-day-header .state-breakdown .state-done { +.state-breakdown .state-done { color: var(--ok); } -.queue-day-header .state-breakdown .state-done::before { +.state-breakdown .state-done::before { background: var(--ok); } -.queue-day-header .state-breakdown .state-failed { +.state-breakdown .state-failed { color: var(--err-text); border-color: oklch(0.62 0.20 28 / 0.30); background: oklch(0.62 0.20 28 / 0.06); } -.queue-day-header .state-breakdown .state-failed::before { +.state-breakdown .state-failed::before { background: var(--err); } -.queue-day-header .state-breakdown .state-gone { +.state-breakdown .state-gone { color: var(--warn); } -.queue-day-header .state-breakdown .state-gone::before { +.state-breakdown .state-gone::before { background: var(--warn); } .queue-day-header .caret { color: var(--muted); font-size: 12px; } @@ -792,6 +792,32 @@ main { .queue-day-stale .queue-day-header .meta { opacity: 0.6; } .queue-day-body { padding: 8px 16px 16px; } +/* Downloads — hour tier nested inside a day body */ +.queue-hours { display: flex; flex-direction: column; gap: 2px; } +.queue-hour { + border-left: 2px solid var(--border); + margin-left: 6px; +} +.queue-hour-header { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 10px; + cursor: pointer; + border-radius: 6px; +} +.queue-hour-header:hover { background: rgba(255, 255, 255, 0.015); } +.queue-hour-header .caret { color: var(--muted); font-size: 12px; } +.queue-hour-header .qh-check { margin: 0; cursor: pointer; } +.queue-hour-header .qh-check:disabled { cursor: not-allowed; opacity: 0.4; } +.queue-hour-header .hour-label { + font-variant-numeric: tabular-nums; + font-weight: 600; + min-width: 104px; +} +.queue-hour-header .meta { flex: 0 0 auto; color: var(--muted); font-size: 12px; } +.queue-hour-body { padding: 4px 0 8px 18px; } + .queue-items { width: 100%; border-collapse: collapse; } .queue-items th, .queue-items td { padding: 6px 8px; text-align: left; @@ -1353,6 +1379,13 @@ main { -webkit-overflow-scrolling: touch; width: 100%; } + .queue-hour-body { padding: 4px 0 8px 8px; } + .queue-hour-body table.queue-items { + display: block; + overflow-x: auto; + -webkit-overflow-scrolling: touch; + width: 100%; + } .exports-table-wrap, .exports-panel { overflow-x: auto; } .exports-table { min-width: 520px; } @@ -1361,7 +1394,9 @@ main { * crowding the row. */ .queue-day-header { flex-wrap: wrap; } .queue-day-header h3 { flex: 1 1 auto; } - .queue-day-header .state-breakdown { + .queue-hour-header { flex-wrap: wrap; } + .queue-hour-header .hour-label { flex: 1 1 auto; } + .state-breakdown { flex: 1 1 100%; justify-content: flex-start; margin-top: 4px; From 7023484e1c342830a8f1829f18a46dcda600dc5d Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 4 Jun 2026 22:11:00 +0100 Subject: [PATCH 14/21] feat(exports): refresh the export jobs list UI Rework the export jobs table to be clearer and tidier: - Drop the ID and Created columns. - Show export type as a human-readable badge (Join Front / Join Rear / PiP Fr / PiP Rf), echoing the toolbar buttons. - Merge State + Progress into a single Status cell with a slim inline progress bar for running jobs. - Add a Footage column showing the source clips' date range and clip count, captured onto the export_jobs row at enqueue time (new clip_start/clip_end columns) so it survives retention pruning of the underlying clips. - Replace the text Download/Delete links with right-aligned icon buttons; the delete bin sits in a fixed rightmost column. Co-Authored-By: Claude Opus 4.8 --- tests/test_export_clip_range.py | 127 +++++++++++++++++++++++ tests/test_export_download_filename.py | 31 ++++++ web/db.py | 10 +- web/routers/exports.py | 12 ++- web/services/exporter.py | 16 ++- web/static/app.js | 134 ++++++++++++++++++++----- web/static/styles.css | 70 ++++++++++++- 7 files changed, 370 insertions(+), 30 deletions(-) create mode 100644 tests/test_export_clip_range.py diff --git a/tests/test_export_clip_range.py b/tests/test_export_clip_range.py new file mode 100644 index 0000000..7ac87d5 --- /dev/null +++ b/tests/test_export_clip_range.py @@ -0,0 +1,127 @@ +"""Export jobs capture the source clips' date range at creation. + +The export jobs list UI shows the date range of the footage in each +export. The source clips get pruned by retention over time, so the +min/max clip timestamp is snapshotted onto the ``export_jobs`` row +when the job is enqueued rather than derived on every list load. +""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.services.exporter import ExportWorker + +# export_jobs schema as it shipped before clip_start/clip_end existed. +_OLD_EXPORT_JOBS = """ +CREATE TABLE export_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + clip_ids TEXT NOT NULL, + state TEXT NOT NULL, + progress REAL NOT NULL DEFAULT 0.0, + output_path TEXT, + error TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + finished_at INTEGER +); +""" + + +@pytest.fixture +def db(tmp_path: Path) -> Database: + return Database(str(tmp_path / "test.db")) + + +def _insert_clip(db: Database, clip_id: int, ts: int, camera: str) -> None: + with db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(id, path, basename, group_name, timestamp, camera, " + " sequence, event_type, has_gpx, gps_examined, scanned_at) " + "VALUES (?,?,?,?,?,?,?,?,0,0,?)", + (clip_id, f"/rec/{clip_id}.mp4", f"{clip_id}.mp4", + "2024-03-15", ts, camera, clip_id, "normal", ts), + ) + + +async def _async_noop(_event): # pragma: no cover — broadcast stub + pass + + +def test_existing_db_gains_clip_range_columns(tmp_path: Path) -> None: + """A pre-existing DB with the old export_jobs schema is migrated to + carry clip_start/clip_end, and its existing rows survive.""" + path = tmp_path / "old.db" + raw = sqlite3.connect(str(path)) + raw.executescript(_OLD_EXPORT_JOBS) + raw.execute( + "INSERT INTO export_jobs (type, clip_ids, state, created_at) " + "VALUES ('join_front', '{\"clip_ids\": [1]}', 'queued', 5)" + ) + raw.commit() + raw.close() + + db = Database(str(path)) + with db.conn() as c: + cols = { + r["name"] + for r in c.execute("PRAGMA table_info(export_jobs)") + } + row = c.execute( + "SELECT type, clip_start, clip_end FROM export_jobs" + ).fetchone() + + assert "clip_start" in cols + assert "clip_end" in cols + # Existing row preserved; the new columns default to NULL. + assert row["type"] == "join_front" + assert row["clip_start"] is None + assert row["clip_end"] is None + + +def test_enqueue_snapshots_clip_date_range(db: Database, monkeypatch) -> None: + """enqueue records the min/max timestamp of the selected clips so + the range survives even after the clips are retention-pruned.""" + monkeypatch.setattr( + "web.services.exporter.ffmpeg_available", lambda: True + ) + _insert_clip(db, 1, 1_700_000_500, "F") + _insert_clip(db, 2, 1_700_000_100, "F") # earliest + _insert_clip(db, 3, 1_700_000_900, "F") # latest + + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_async_noop) + job_id = worker.enqueue("join_front", [1, 2, 3]) + + with db.conn() as c: + row = c.execute( + "SELECT clip_start, clip_end FROM export_jobs WHERE id=?", + (job_id,), + ).fetchone() + assert row["clip_start"] == 1_700_000_100 + assert row["clip_end"] == 1_700_000_900 + + +def test_enqueue_stores_null_range_for_unknown_clips( + db: Database, monkeypatch +) -> None: + """If none of the selected clips resolve in clip_index (e.g. a + stale selection), the range columns stay NULL rather than 500ing.""" + monkeypatch.setattr( + "web.services.exporter.ffmpeg_available", lambda: True + ) + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_async_noop) + job_id = worker.enqueue("pip", [90, 91]) + + with db.conn() as c: + row = c.execute( + "SELECT clip_start, clip_end FROM export_jobs WHERE id=?", + (job_id,), + ).fetchone() + assert row["clip_start"] is None + assert row["clip_end"] is None diff --git a/tests/test_export_download_filename.py b/tests/test_export_download_filename.py index fd807fe..93bb279 100644 --- a/tests/test_export_download_filename.py +++ b/tests/test_export_download_filename.py @@ -111,6 +111,37 @@ def test_download_uses_derived_filename(logged_in_client, assert expected in r.headers["content-disposition"] +def _insert_job_with_range(db, job_type, clip_ids, clip_start, clip_end): + import json + with db.write() as c: + cur = c.execute( + "INSERT INTO export_jobs " + "(type, clip_ids, state, created_at, clip_start, clip_end) " + "VALUES (?,?, 'done', 1, ?, ?)", + (job_type, json.dumps({"clip_ids": clip_ids, + "encoder": "software"}), + clip_start, clip_end), + ) + return cur.lastrowid + + +def test_list_jobs_returns_clip_range_and_count(logged_in_client): + """GET /api/exports surfaces the stored footage date range and a + clip count derived from clip_ids, so the UI can render the + 'Footage' column without per-row clip lookups.""" + db = logged_in_client.app.state.db + ts1 = _ts(2024, 3, 15, 14, 30) + ts2 = _ts(2024, 3, 15, 15, 2) + job_id = _insert_job_with_range(db, "join_front", [1, 2], ts1, ts2) + + r = logged_in_client.get("/api/exports") + assert r.status_code == 200 + job = next(j for j in r.json()["jobs"] if j["id"] == job_id) + assert job["clip_start"] == ts1 + assert job["clip_end"] == ts2 + assert job["clip_count"] == 2 + + def test_download_falls_back_when_clips_pruned(logged_in_client, tmp_recordings_dir): """A done job whose source clips were retention-pruned still diff --git a/web/db.py b/web/db.py index 76af657..c2048c7 100644 --- a/web/db.py +++ b/web/db.py @@ -144,7 +144,9 @@ def migrate_legacy_db_path(new_path: str) -> None: error TEXT, created_at INTEGER NOT NULL, started_at INTEGER, - finished_at INTEGER + finished_at INTEGER, + clip_start INTEGER, -- min source-clip timestamp (unix s) + clip_end INTEGER -- max source-clip timestamp (unix s) ); CREATE TABLE IF NOT EXISTS kv ( @@ -215,6 +217,12 @@ def _add_column(table: str, col: str, ddl: str) -> None: "WHERE has_gpx = 1 AND gps_examined = 0" ) + # Date range of an export's source clips, snapshotted at + # enqueue time so the export list can show it after the + # underlying clips are retention-pruned. + _add_column("export_jobs", "clip_start", "INTEGER") + _add_column("export_jobs", "clip_end", "INTEGER") + @contextmanager def conn(self) -> Iterator[sqlite3.Connection]: """Yield a connection with row-factory set. diff --git a/web/routers/exports.py b/web/routers/exports.py index cb0bf95..323b5fe 100644 --- a/web/routers/exports.py +++ b/web/routers/exports.py @@ -97,10 +97,18 @@ def list_jobs(request: Request) -> JSONResponse: with request.app.state.db.conn() as c: rows = c.execute( "SELECT id, type, state, progress, error, " - "created_at, started_at, finished_at " + "created_at, started_at, finished_at, " + "clip_start, clip_end, clip_ids " "FROM export_jobs ORDER BY created_at DESC LIMIT 100" ).fetchall() - return JSONResponse({"jobs": [dict(r) for r in rows]}) + jobs = [] + for r in rows: + job = dict(r) + # clip_count is derived from the always-present clip_ids; the + # raw id list isn't useful to the UI, so swap it out. + job["clip_count"] = len(parse_clip_ids(job.pop("clip_ids"))) + jobs.append(job) + return JSONResponse({"jobs": jobs}) @router.get("/{job_id}/download") diff --git a/web/services/exporter.py b/web/services/exporter.py index de2ad36..f93bc36 100644 --- a/web/services/exporter.py +++ b/web/services/exporter.py @@ -276,13 +276,23 @@ def enqueue( "encoder": encoder, }) with self.db.write() as c: + # Snapshot the footage date range now — the source clips + # may be retention-pruned long before the export is. + ph = ",".join("?" * len(clip_ids)) + rng = c.execute( + f"SELECT MIN(timestamp) AS lo, MAX(timestamp) AS hi " + f"FROM clip_index WHERE id IN ({ph})", + clip_ids, + ).fetchone() cur = c.execute( """ INSERT INTO export_jobs - (type, clip_ids, state, created_at) - VALUES (?, ?, 'queued', ?) + (type, clip_ids, state, created_at, + clip_start, clip_end) + VALUES (?, ?, 'queued', ?, ?, ?) """, - (job_type, payload, int(time.time())), + (job_type, payload, int(time.time()), + rng["lo"], rng["hi"]), ) return cur.lastrowid diff --git a/web/static/app.js b/web/static/app.js index 3a374e5..6417a59 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -1164,6 +1164,64 @@ function updateExportsSummary(jobs) { : "Export jobs"; } +// Human-readable export type labels. These echo the toolbar +// buttons: Join F/R and the PiP Fr/Rf (front-main / rear-main). +const EXPORT_TYPE_LABELS = { + join_front: "Join Front", + join_rear: "Join Rear", + pip: "PiP Fr", + pip_rear: "PiP Rf", +}; + +// Heroicons solid (MIT) — arrow-down-tray (download) + trash (delete), +// matching the inline-SVG / currentColor pattern used in the header. +const EXPORT_ICON_DOWNLOAD = + ''; + +const EXPORT_ICON_TRASH = + ''; + +function escapeExportText(s) { + const d = document.createElement("div"); + d.textContent = String(s); + return d.innerHTML; +} + +// "15 Mar 14:30–15:02" (same day), "15 Mar – 17 Mar" (spans days), +// or "—" when no range was captured (jobs predating the feature, or +// clips that couldn't be resolved at enqueue time). +function formatExportRange(start, end) { + if (!start) return "—"; + const s = new Date(start * 1000); + const e = new Date((end || start) * 1000); + const dOpts = { day: "numeric", month: "short" }; + const tOpts = { hour: "2-digit", minute: "2-digit", hour12: false }; + if (s.toDateString() === e.toDateString()) { + const day = s.toLocaleDateString([], dOpts); + const st = s.toLocaleTimeString([], tOpts); + const et = e.toLocaleTimeString([], tOpts); + return st === et ? `${day} ${st}` : `${day} ${st}–${et}`; + } + return `${s.toLocaleDateString([], dOpts)} – ` + + `${e.toLocaleDateString([], dOpts)}`; +} + function renderExportJobs(jobs) { updateExportsSummary(jobs); const el = document.getElementById("exports-list"); @@ -1177,8 +1235,8 @@ function renderExportJobs(jobs) { table.className = "exports-table"; table.innerHTML = ` - IDTypeStateProgress - Created + TypeStatusFootage + `; @@ -1186,35 +1244,65 @@ function renderExportJobs(jobs) { const live = state.exportProgress || {}; for (const j of jobs) { const tr = document.createElement("tr"); - const created = j.created_at - ? new Date(j.created_at * 1000).toLocaleString() : "—"; // Running jobs: prefer the live progress stream if we have // one; the DB row is only updated at finish. const liveHit = live[j.id]; const progVal = liveHit && liveHit.progress != null ? liveHit.progress : j.progress; - const terminal = ["done", "failed", "cancelled"].includes(j.state); - let pct; - if (j.state === "done") pct = "100%"; - else if (terminal) pct = "—"; - else pct = progVal != null ? Math.round(progVal * 100) + "%" : "—"; - const stage = liveHit && liveHit.stage && !terminal - ? ` · ${liveHit.stage}` : ""; - const actions = []; - if (j.state === "done") { - actions.push( - `
Download`, - ); + + // Type badge. + const label = EXPORT_TYPE_LABELS[j.type] || j.type; + const typeCell = + `${escapeExportText(label)}`; + + // Status: state text, plus an inline progress bar while running + // and the error message on failure. + let statusCell = `${j.state}`; + if (j.state === "failed" && j.error) { + statusCell += + ` · ${escapeExportText(j.error)}`; + } + if (j.state === "running") { + const pct = progVal != null ? Math.round(progVal * 100) : 0; + const stage = liveHit && liveHit.stage ? ` · ${liveHit.stage}` : ""; + statusCell += + `
` + + `
` + + `
${pct}%` + + `${escapeExportText(stage)}`; } - actions.push(``); + + // Footage: captured date range + clip count. + const range = formatExportRange(j.clip_start, j.clip_end); + const n = j.clip_count || 0; + const footageCell = range === "—" + ? '' + : `${escapeExportText(range)}` + + (n ? ` · ${n} ` + + `clip${n === 1 ? "" : "s"}` : ""); + + // Actions: download (when ready) + delete. The download slot is + // always reserved — an invisible placeholder when there's no + // download — so the bin stays pinned to the right and never + // jumps across as jobs finish. + const dl = j.state === "done" + ? `` + + `${EXPORT_ICON_DOWNLOAD}` + : ``; + const del = + ``; + tr.innerHTML = ` - ${j.id} - ${j.type} - ${j.state}${j.error ? " · " + j.error : ""} - ${pct}${stage} - ${created} - ${actions.join(" · ")} + ${typeCell} + ${statusCell} + ${footageCell} + ${dl}${del} `; tbody.appendChild(tr); } diff --git a/web/static/styles.css b/web/static/styles.css index 78d3c9f..60625d4 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -407,14 +407,82 @@ main { } .exports-table { width: 100%; border-collapse: collapse; } .exports-table th, .exports-table td { - padding: 6px 10px; text-align: left; + padding: 7px 10px; text-align: left; vertical-align: middle; border-bottom: 1px solid var(--border); font-size: 13px; } .exports-table th { color: var(--muted); font-weight: 500; } +.exports-table tbody tr:last-child td { border-bottom: none; } +.exports-table tbody tr:hover { background: var(--panel-2); } .state-queued { color: var(--muted); } .state-running { color: var(--accent); } .state-cancelled { color: var(--warn); } +/* Type badge — small accent-tinted pill. */ +.export-type { + display: inline-block; + padding: 2px 8px; + border-radius: 999px; + background: var(--accent-10); + border: 1px solid var(--accent-35); + color: var(--accent); + font-size: 12px; + font-weight: 500; + white-space: nowrap; +} + +/* Status cell — state text + slim inline progress bar. */ +.export-status .export-err { color: var(--err-text); } +.export-progress { + margin-top: 5px; + height: 4px; + width: 100%; + max-width: 160px; + background: var(--panel-3); + border-radius: 999px; + overflow: hidden; +} +.export-progress-fill { + height: 100%; + background: var(--accent); + border-radius: inherit; + transition: width 200ms ease; +} +.export-stage { + display: block; + margin-top: 3px; + font-size: 11px; + color: var(--muted); +} + +/* Footage cell — date range + clip count. */ +.export-footage { white-space: nowrap; } +.export-count { color: var(--muted); } + +/* Actions — right-aligned icon buttons; the trailing delete button + * lines up in a column across rows, with download just left of it. */ +.exports-actions-col { width: 1%; } +.export-actions { text-align: right; white-space: nowrap; } +.export-action { + display: inline-flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + padding: 0; + border: 1px solid transparent; + border-radius: 6px; + background: transparent; + color: var(--muted); + cursor: pointer; + vertical-align: middle; + text-decoration: none; +} +.export-action + .export-action { margin-left: 4px; } +.export-action:hover { background: var(--panel-3); color: var(--text); } +.export-delete:hover { color: var(--err-text); } +/* Reserves the download slot so the bin never shifts. */ +.export-action--empty { visibility: hidden; cursor: default; } + /* Day cards */ .day { background: var(--panel); From a37a10db58a4cc496c0d01e465bb199e214a7cf5 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Thu, 4 Jun 2026 22:11:13 +0100 Subject: [PATCH 15/21] fix(exports): write absolute clip paths into the concat list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ffmpeg's concat demuxer resolves relative entries in the list file against the directory of the list file itself — which lives in the system temp dir. When clip_index stores relative paths (a dev box launched with a relative RECORDINGS), a join export fed ffmpeg a relative path and it went looking under /tmp/.../recordings/..., failing with "No such file or directory". Resolve each clip path to an absolute path before writing it, so the concat works regardless of how the path was stored or where the temp list file happens to live. PiP exports were unaffected (they list absolute mkdtemp segment paths). Co-Authored-By: Claude Opus 4.8 --- tests/test_exporter_concat_paths.py | 64 +++++++++++++++++++++++++++++ web/services/exporter.py | 9 +++- 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/test_exporter_concat_paths.py diff --git a/tests/test_exporter_concat_paths.py b/tests/test_exporter_concat_paths.py new file mode 100644 index 0000000..2a7b8cd --- /dev/null +++ b/tests/test_exporter_concat_paths.py @@ -0,0 +1,64 @@ +"""The join concat demuxer must list clips by ABSOLUTE path. + +ffmpeg's concat demuxer resolves *relative* entries in the list file +against the directory of the list file itself — which lives in the +system temp dir. So when clip_index stores relative paths (a dev box +launched with a relative ``RECORDINGS``), a relative entry sends +ffmpeg looking under ``/tmp/.../recordings/...`` and the export dies +with "No such file or directory". Writing absolute paths makes the +concat robust regardless of how the path was stored, or where the +temp list file happens to live. +""" +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.services.exporter import ExportWorker + + +@pytest.fixture +def db(tmp_path: Path) -> Database: + return Database(str(tmp_path / "test.db")) + + +async def _async_noop(_event): # pragma: no cover — broadcast stub + pass + + +async def test_concat_list_uses_absolute_paths( + db: Database, tmp_path: Path, monkeypatch +) -> None: + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_async_noop) + captured: dict = {} + + async def fake_probe(_clips): + return 1.0 + + async def fake_ffmpeg(_job_id, args, _total): + # Read the concat list while it still exists (-i ), + # then touch the output so the rc==0 branch marks it done. + list_file = args[args.index("-i") + 1] + captured["lines"] = Path(list_file).read_text().splitlines() + Path(args[-1]).write_bytes(b"\0") + return 0, "" + + monkeypatch.setattr(worker, "_probe_total", fake_probe) + monkeypatch.setattr(worker, "_run_ffmpeg", fake_ffmpeg) + + # A relative clip path, exactly as stored when RECORDINGS is given + # relative on a local dev box. + rel = "recordings/2026-06-04/clip_F.MP4" + out = str(tmp_path / "out.mp4") + await worker._concat(1, [{"path": rel}], out) + + assert captured.get("lines"), "ffmpeg was never invoked" + for line in captured["lines"]: + # Each entry is: file '' + assert line.startswith("file '/"), \ + f"concat entry is not absolute: {line!r}" + assert f"file '{os.path.abspath(rel)}'" in captured["lines"] diff --git a/web/services/exporter.py b/web/services/exporter.py index f93bc36..ac70611 100644 --- a/web/services/exporter.py +++ b/web/services/exporter.py @@ -460,8 +460,13 @@ async def _concat( ) as f: list_file = f.name for c in clips: - # Escape single quotes per ffmpeg concat docs. - safe = c["path"].replace("'", "'\\''") + # Absolute path: ffmpeg's concat demuxer resolves + # relative entries against the list file's own + # directory (the temp dir), not our CWD — so a + # relative clip path (dev boxes with a relative + # RECORDINGS) would send ffmpeg looking in /tmp. + # Escape single quotes per the ffmpeg concat docs. + safe = os.path.abspath(c["path"]).replace("'", "'\\''") f.write(f"file '{safe}'\n") try: From c47258366c169e3fe91d6d7b2ffdc2f2f6841269 Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Fri, 5 Jun 2026 21:04:32 +0100 Subject: [PATCH 16/21] feat(import): manual SD-card / USB import Bring Viofo clips into the archive without Wi-Fi sync, via browser folder upload (with folder drag-and-drop) or a configurable folder/USB drop path. Imported clips land in the same recordings// layout with GPX, thumbnails, indexing, RO/parking classification and retention, reusing the post-sync pipeline. - IMPORT_PATH advanced setting (default /import) - importer service: scan_source, ingest_clip (dedup, cross-volume copy+verify keeping the source, same-volume restore-on-failure), run_folder_ingest with hub progress - smart quota eviction that never deletes anything newer than the clip being imported; import staging excluded from the quota walk - /api/import scan|ingest|upload endpoints (auth + CSRF, path-safe) - RO/locked status survives rescans via a download_queue origin row - Import modal (Upload + Folder tabs) with folder drag-and-drop, launched from the Download manager - README + CHANGELOG Squashed from the feature/sd-card-import branch. Also folds in the v2.2 README/CHANGELOG release-notes edits that were in progress on the branch. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 35 +++ README.md | 35 ++- tests/test_import_endpoints.py | 138 ++++++++++ tests/test_importer.py | 213 +++++++++++++++ tests/test_make_room_for.py | 175 +++++++++++++ tests/test_retention.py | 2 +- tests/test_settings_schema.py | 13 + tests/test_sync_worker_retention_loop.py | 1 + web/app.py | 2 + web/routers/imports.py | 158 +++++++++++ web/routers/settings.py | 1 + web/services/importer.py | 319 +++++++++++++++++++++++ web/services/retention.py | 133 ++++++++-- web/services/sync_worker.py | 6 + web/settings.py | 2 + web/settings_schema.py | 8 +- web/static/app.js | 220 ++++++++++++++++ web/static/index.html | 67 +++++ web/static/styles.css | 160 ++++++++++++ 19 files changed, 1667 insertions(+), 21 deletions(-) create mode 100644 tests/test_import_endpoints.py create mode 100644 tests/test_importer.py create mode 100644 tests/test_make_room_for.py create mode 100644 web/routers/imports.py create mode 100644 web/services/importer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 23d7d57..4bef2d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +### Added +- Manual import: bring Viofo clips into the archive without Wi-Fi sync, + via browser folder upload or a configurable folder/USB drop path + (`IMPORT_PATH`). Imported clips get the usual GPX, thumbnails, indexing, + RO/parking classification, and retention; quota imports make room as they + go without deleting anything newer than what's being imported. External + sources (USB/SD) are only ever read, never modified. + +## v2.2 — 2026-06-04 + ### Added - Export downloads now use sensible filenames derived from the selected clips' date range, camera, and clip count @@ -41,6 +51,15 @@ the session moving average. To avoid flooding HA it publishes first at ~30 s into a session then at most once per minute, and reports `0` when idle. Enabled by default. +- **Retry failed** button in the web UI download manager re-queues every + failed download in one click (mirrors the existing HA action button). +- Live disk usage is shown in Settings → Archive Retention so you can + see headroom against the retention and critical thresholds. +- Quota-bound retention via the new `RECORDINGS_QUOTA_GB` setting: when + set, `RETENTION_DISK_PCT` and `DISK_CRITICAL_PCT` are measured against + the declared quota instead of the filesystem reported by `statvfs`. + Needed when the recordings directory lives inside a quota-bound share + (Synology shared folder, ZFS dataset quota, etc.). ### Changed - Unified `sync_status` to four states: `downloading`, `waiting`, @@ -52,6 +71,22 @@ update them: `idle` and `stopped` map to `waiting` (or `paused` when sync is fully stopped). Connection state is still reflected via the existing `binary_sensor.viofosync_dashcam`. +- Redesigned the export jobs panel: the type is shown as a + human-readable badge (Join Front / Join Rear / PiP Fr / PiP Rf), + the State and Progress columns are merged into one Status cell with + an inline progress bar, and a new Footage column shows the source + clips' date range and clip count. Download and delete are now icon + buttons. (The ID and Created columns were dropped.) +- The download list is now grouped by hour. + +### Fixed +- Archive retention caps (max age / max clips) are enforced on a + periodic loop rather than only after a download, so they apply even + when no new clips are arriving. +- Join exports could fail with "No such file or directory" when clip + paths were stored relative (e.g. a dev box launched with a relative + `RECORDINGS`): ffmpeg's concat demuxer resolved them against its temp + directory. The concat list now uses absolute paths. ## v2.1 — 2026-05-16 diff --git a/README.md b/README.md index 3efb98c..2c5f54b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Self-hosted web app for syncing, browsing, and exporting recordings from a Viofo - **Archive browser** — clips grouped by day, paired front/rear, in-browser playback. - **GPS journeys** — map per trip with auto-split stops and reverse-geocoded place names. - **Exports** — joined front/rear or picture-in-picture (front-main or rear-main) videos via ffmpeg, with download names derived from the clip date range and camera. Or grab the **original** front/rear clips directly, un-joined. -- **Download manager** — live progress and a reorderable queue. +- **Download manager** — live progress with session speed and ETA, a reorderable queue, hourly grouping, and one-click retry of failed downloads. - **Auto-delete from dashcam** *(optional)* — frees SD card space once a clip is safely downloaded. - **Settings page** — runtime settings hot-reload; no Docker env vars to fiddle with. - **Home Assistant** *(optional)* — auto-discovered sensors and buttons via MQTT. @@ -58,6 +58,37 @@ The only Docker-level env vars are: App-level settings (sync interval, dashcam IP, encoder, geocoding email, web port, retention, password, auto-delete, etc.) are editable on the **Settings** page. Advanced users can hand-edit `/config/config.json` between restarts; the schema lives in `[web/settings_schema.py](web/settings_schema.py)`. +### Importing without Wi-Fi + +Use **Import manually** in the web UI to ingest clips you already have on +disk — no dashcam connection needed. Two modes: + +- **Upload** — pick a folder in your browser; clips upload one at a time + and slot straight into the archive. On a quota-bound archive it makes + room as it goes, evicting the oldest clips (never anything newer than + what you're importing). +- **Folder** — copy clips into the `import` folder inside your recordings + share, then **Scan** → **Ingest**. By default this is `recordings/import`; + for a one-off import from a different path, type it in the Import dialog's + Folder tab, or set a persistent default via the advanced `IMPORT_PATH` key + in `/config/config.json`. + +**From a USB drive / card reader:** bind-mount it into the container and set +the import path to the mount, e.g.: + + docker run ... -v /mnt/usb:/import robxyz/viofosync + # then type /import in the Import dialog, or set IMPORT_PATH=/import in /config/config.json + +The source is only ever **read** — originals on the card/USB are never +deleted. If you plug the drive in *after* the container starts, either +restart the container or use shared mount propagation +(`-v /mnt:/mnt:rshared`, with the host mount also shared) so the container +sees it. + +Imported clips are recognised by Viofo naming +(`YYYY_MMDD_HHMMSS_NNNN[event][cam].MP4`); locked clips under an `RO/` +folder keep their protected status. Non-matching files are left untouched. + ## Alternative camera address You can set an optional **Alternative address** (Settings → Dashcam) — a @@ -105,7 +136,7 @@ When MQTT is on, viofosync publishes: Enabled by default in HA: dashcam connectivity, dashcam connection (`primary` / `alternative` / `offline`, with the live address as an `address` attribute), sync status -(`stopped` / `idle` / `paused` / `downloading`), queue pending, last +(`downloading` / `waiting` / `paused` / `error`), queue pending, last downloaded clip, disk used, and six action buttons (start/pause/skip/refresh/retry-failed/rescan). diff --git a/tests/test_import_endpoints.py b/tests/test_import_endpoints.py new file mode 100644 index 0000000..c30a46e --- /dev/null +++ b/tests/test_import_endpoints.py @@ -0,0 +1,138 @@ +"""Import endpoint tests (auth, scan, ingest, upload).""" +from __future__ import annotations + +from pathlib import Path + +import pytest + + +class _FakeMqttService: + def __init__(self, **kwargs): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): + return {"state": "idle", "detail": None, "last_published_at": None} + + +@pytest.fixture +def client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) + + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + # CSRF: token comes from GET /api/auth/csrf and rides the + # x-csrf-token header (bound to the session cookie), not a cookie. + csrf = c.get("/api/auth/csrf").json()["csrf"] + c.headers.update({"x-csrf-token": csrf}) + yield c, Path(tmp_recordings_dir) + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def test_scan_requires_session(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) + app = create_app() + with TestClient(app) as c: + r = c.post("/api/import/scan", json={}) + assert r.status_code in (401, 403) + settings_mod.reset_for_tests() + + +def test_scan_lists_recognised_and_skipped(client): + c, rec = client + card = rec / "import" / "DCIM" + card.mkdir(parents=True) + (card / "2026_0101_080000_0001F.MP4").write_bytes(b"a" * 10) + (card / "junk.bin").write_bytes(b"z") + r = c.post("/api/import/scan", json={}) + assert r.status_code == 200 + body = r.json() + assert body["total_bytes"] == 10 + assert [it["basename"] for it in body["recognised"]] == [ + "2026_0101_080000_0001F.MP4"] + assert {s["name"] for s in body["skipped"]} == {"junk.bin"} + + +def test_scan_bad_path_400(client): + c, rec = client + r = c.post("/api/import/scan", json={"path": str(rec / "nope")}) + assert r.status_code == 400 + + +def test_upload_writes_clip_into_archive(client): + c, rec = client + name = "2026_0101_080000_0001F.MP4" + r = c.post( + "/api/import/upload", + content=b"a" * 12, + headers={ + "X-Import-Path": f"DCIM/Movie/{name}", + "X-Import-Size": "12", + "Content-Type": "application/octet-stream", + }, + ) + assert r.status_code == 200 + assert r.json()["status"] == "imported" + assert (rec / "2026-01-01" / name).exists() + + +def test_upload_rejects_non_viofo_name(client): + c, rec = client + r = c.post( + "/api/import/upload", + content=b"x", + headers={"X-Import-Path": "DCIM/whatever.mp4", "X-Import-Size": "1"}, + ) + assert r.status_code == 200 + assert r.json()["status"] == "not_recognised" + + +def test_ingest_bad_path_400(client): + c, rec = client + r = c.post("/api/import/ingest", json={"path": str(rec / "nope")}) + assert r.status_code == 400 + + +def test_ingest_409_when_already_running(client): + c, rec = client + c.app.state.import_running = True + try: + r = c.post("/api/import/ingest", json={}) + assert r.status_code == 409 + finally: + c.app.state.import_running = False diff --git a/tests/test_importer.py b/tests/test_importer.py new file mode 100644 index 0000000..522431c --- /dev/null +++ b/tests/test_importer.py @@ -0,0 +1,213 @@ +"""Importer core tests.""" +from __future__ import annotations + +import os +import types +from pathlib import Path + + +def test_classify_event_type(): + from web.services import importer + assert importer.classify_event_type("F", "DCIM/Movie/X.MP4") == "normal" + assert importer.classify_event_type("PF", "DCIM/Parking/X.MP4") == "parking" + assert importer.classify_event_type("F", "DCIM/Movie/RO/X.MP4") == "ro" + # RO wins even for a parking-named clip living under /RO/. + assert importer.classify_event_type("PF", "Movie/RO/X.MP4") == "ro" + + +def test_scan_source_recurses_and_sorts_newest_first(tmp_path: Path): + from web.services import importer + root = tmp_path / "card" + (root / "DCIM" / "Movie").mkdir(parents=True) + (root / "DCIM" / "Movie" / "RO").mkdir() + # Recognised: one older normal front, one newer locked front. + (root / "DCIM" / "Movie" / "2026_0101_080000_0001F.MP4").write_bytes(b"a" * 10) + (root / "DCIM" / "Movie" / "RO" / "2026_0102_090000_0002F.MP4").write_bytes(b"b" * 20) + # Junk: ignored + reported. + (root / "DCIM" / "Movie" / "notes.txt").write_text("hi") + # Matches the filename regex but has an impossible date -> bad_timestamp. + (root / "DCIM" / "Movie" / "2026_1399_250000_0009F.MP4").write_bytes(b"c" * 5) + + manifest = importer.scan_source(str(root)) + assert manifest.total_bytes == 30 + assert [it.basename for it in manifest.items] == [ + "2026_0102_090000_0002F.MP4", # newest first + "2026_0101_080000_0001F.MP4", + ] + assert manifest.items[0].event_type == "ro" + assert manifest.items[1].event_type == "normal" + skipped = {s["name"]: s["reason"] for s in manifest.skipped} + assert skipped["notes.txt"] == "not_recognised" + assert skipped["2026_1399_250000_0009F.MP4"] == "bad_timestamp" + + +def _snap(rec: Path, **over): + base = dict( + recordings=str(rec), grouping="daily", gps_extract=False, + retention_disk_pct=0, recordings_quota_gb=0, retention_protect_ro=True, + retention_max_days=0, import_path="", + ) + base.update(over) + return types.SimpleNamespace(**base) + + +def _origin_rows(db): + with db.conn() as c: + return {r["filename"]: dict(r) for r in c.execute( + "SELECT filename, source_dir, event_type, state, manual " + "FROM download_queue").fetchall()} + + +def test_ingest_clip_places_file_and_records_origin(tmp_path: Path): + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + src = tmp_path / "card" / "RO" + src.mkdir(parents=True) + name = "2026_0102_090000_0002F.MP4" + (src / name).write_bytes(b"b" * 20) + + man = importer.scan_source(str(tmp_path / "card")) + item = man.items[0] + res = importer.ingest_clip(db, _snap(rec), item, cross_volume=False) + + assert res.status == "imported" + dest = rec / "2026-01-02" / name + assert dest.exists() + assert not (src / name).exists() # same-volume move + rows = _origin_rows(db) + assert rows[name]["state"] == "done" + assert rows[name]["manual"] == 1 + assert "/RO/" in rows[name]["source_dir"] # RO survives via origin row + assert rows[name]["event_type"] == "ro" + + +def test_ingest_clip_cross_volume_copies_and_keeps_source(tmp_path: Path): + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + name = "2026_0101_080000_0001F.MP4" + src = tmp_path / "usb" + src.mkdir() + (src / name).write_bytes(b"a" * 10) + man = importer.scan_source(str(src)) + res = importer.ingest_clip(db, _snap(rec), man.items[0], cross_volume=True) + assert res.status == "imported" + assert (rec / "2026-01-01" / name).exists() + assert (src / name).exists() # original kept + rows = _origin_rows(db) + assert rows[name]["state"] == "done" + assert rows[name]["manual"] == 1 + + +def test_ingest_clip_skips_duplicate(tmp_path: Path): + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + name = "2026_0101_080000_0001F.MP4" + (rec / "2026-01-01").mkdir() + (rec / "2026-01-01" / name).write_bytes(b"existing") + src = tmp_path / "usb" + src.mkdir() + (src / name).write_bytes(b"a" * 10) + man = importer.scan_source(str(src)) + res = importer.ingest_clip(db, _snap(rec), man.items[0], cross_volume=True) + assert res.status == "already_present" + assert (rec / "2026-01-01" / name).read_bytes() == b"existing" + + +def test_ingest_clip_restores_source_when_final_rename_fails(tmp_path, monkeypatch): + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + src = tmp_path / "card" + src.mkdir() + name = "2026_0101_080000_0001F.MP4" + (src / name).write_bytes(b"a" * 10) + man = importer.scan_source(str(src)) + + real_replace = os.replace + calls = {"n": 0} + def flaky_replace(a, b): + calls["n"] += 1 + if calls["n"] == 2: # 1st = src->staging, 2nd = staging->dest + raise OSError("disk full") + return real_replace(a, b) + monkeypatch.setattr(importer.os, "replace", flaky_replace) + + res = importer.ingest_clip(db, _snap(rec), man.items[0], cross_volume=False) + assert res.status == "error" + # Source must be restored (no data loss); dest must not exist. + assert (src / name).exists() + assert not (rec / "2026-01-01" / name).exists() + + +class _FakeHub: + def __init__(self): + self.events = [] + + def schedule_broadcast(self, loop, event): + self.events.append(event) + + +def test_run_folder_ingest_imports_and_summarises(tmp_path: Path): + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + card = tmp_path / "card" / "DCIM" / "Movie" + card.mkdir(parents=True) + (card / "2026_0101_080000_0001F.MP4").write_bytes(b"a" * 10) + (card / "2026_0101_080000_0001R.MP4").write_bytes(b"b" * 10) + (card / "junk.bin").write_bytes(b"z") + + hub = _FakeHub() + summary = importer.run_folder_ingest( + db, _snap(rec), hub, loop=None, root=str(tmp_path / "card"), + ) + assert summary["imported"] == 2 + assert summary["bytes_imported"] == 20 # two 10-byte clips + assert summary["not_recognised"] == 1 + assert (rec / "2026-01-01" / "2026_0101_080000_0001F.MP4").exists() + # clip_index was populated by the post-ingest scan. + with db.conn() as c: + n = c.execute("SELECT COUNT(*) AS n FROM clip_index").fetchone()["n"] + assert n == 2 + types_seen = {e["type"] for e in hub.events} + assert {"import_started", "import_progress", "import_done"} <= types_seen + + +def test_ro_event_type_survives_rescan(tmp_path: Path): + from web.db import Database + from web.services import importer, scanner + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + card = tmp_path / "card" / "DCIM" / "Movie" / "RO" + card.mkdir(parents=True) + name = "2026_0101_080000_0001F.MP4" + (card / name).write_bytes(b"a" * 10) + + importer.run_folder_ingest( + db, _snap(rec), _FakeHub(), loop=None, root=str(tmp_path / "card"), + ) + + def _evt(): + with db.conn() as c: + return c.execute( + "SELECT event_type FROM clip_index WHERE basename=?", + (name,)).fetchone()["event_type"] + + assert _evt() == "ro" + # A second, independent full rescan must keep it 'ro'. + scanner.scan(db, str(rec), "daily") + assert _evt() == "ro" diff --git a/tests/test_make_room_for.py b/tests/test_make_room_for.py new file mode 100644 index 0000000..fc34654 --- /dev/null +++ b/tests/test_make_room_for.py @@ -0,0 +1,175 @@ +"""Tests for import staging exclusion + make_room_for.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from web.db import Database +from web.services import retention as ret + + +@pytest.fixture +def env(tmp_path: Path): + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(tmp_path / ".viofosync.db")) + return rec, db + + +def test_import_exclude_set(tmp_path): + rec = tmp_path / "rec" + rec.mkdir() + rec_abs = str(rec.resolve()) + tmp = f"{rec_abs}/.import_tmp" + + # Default: .import_tmp + the in-tree default import folder. + s = ret.import_exclude_set(rec_abs) + assert s == frozenset({tmp, f"{rec_abs}/import"}) + + # In-tree custom import dir is excluded. + s = ret.import_exclude_set(rec_abs, f"{rec_abs}/incoming") + assert f"{rec_abs}/incoming" in s and tmp in s + + # Out-of-tree (external mount) import path: only .import_tmp excluded. + s = ret.import_exclude_set(rec_abs, "/mnt/usb") + assert s == frozenset({tmp}) + + # import_path equal to the recordings root is NOT added. + s = ret.import_exclude_set(rec_abs, rec_abs) + assert s == frozenset({tmp}) + + +def test_scan_dir_bytes_excludes_given_dirs(env): + rec, _db = env + (rec / "2026-01-01").mkdir() + (rec / "2026-01-01" / "A.MP4").write_bytes(b"x" * 1000) + staging = rec / ".import_tmp" + staging.mkdir() + (staging / "B.MP4").write_bytes(b"y" * 5000) + + everything = ret._scan_dir_bytes(str(rec)) + excluded = ret._scan_dir_bytes( + str(rec), exclude=frozenset({str(staging.resolve())}) + ) + assert everything == 6000 + assert excluded == 1000 + + +def _clip(rec: Path, db: Database, *, basename: str, ts: int, + size: int, event_type: str = "normal") -> int: + import time as _t + day = _t.strftime("%Y-%m-%d", _t.gmtime(ts)) + folder = rec / day + folder.mkdir(exist_ok=True) + (folder / basename).write_bytes(b"x" * size) + with db.write() as c: + cur = c.execute( + "INSERT INTO clip_index " + "(path, basename, group_name, timestamp, camera, sequence, " + " event_type, size_bytes, has_gpx, scanned_at) " + "VALUES (?,?,?,?, 'F', 1, ?, ?, 0, ?)", + (str(folder / basename), basename, day, ts, event_type, + size, int(_t.time())), + ) + return cur.lastrowid + + +def _ids(db: Database) -> set[str]: + with db.conn() as c: + return {r["basename"] for r in c.execute( + "SELECT basename FROM clip_index").fetchall()} + + +def _patch_quota(monkeypatch, used_bytes: int): + # Force _scan_dir_bytes to report a fixed total regardless of disk. + monkeypatch.setattr( + "web.services.retention._scan_dir_bytes", + lambda path, exclude=frozenset(): used_bytes, + ) + + +def test_make_room_no_rules_always_true(env): + rec, db = env + assert ret.make_room_for( + db, str(rec), size=10, before_ts=100, + disk_pct=0, quota_gb=0, protect_ro=True, + ) is True + + +def test_make_room_under_quota_no_eviction(env, monkeypatch): + rec, db = env + _clip(rec, db, basename="OLD.MP4", ts=100, size=1) + _clip(rec, db, basename="MID.MP4", ts=200, size=1) + _patch_quota(monkeypatch, used_bytes=2) # 2 bytes, well under 1 GiB + ok = ret.make_room_for( + db, str(rec), size=0, before_ts=300, + disk_pct=0, quota_gb=1, protect_ro=True, + ) + # Under threshold -> nothing evicted, returns True. + assert ok is True + assert _ids(db) == {"OLD.MP4", "MID.MP4"} + + +def test_make_room_skips_when_only_newer_remain(env, monkeypatch): + rec, db = env + _clip(rec, db, basename="NEW.MP4", ts=500, size=1) + # Over quota forever (patched huge); importing an OLDER clip (ts=100) + # can't evict NEW (newer) -> must skip. + _patch_quota(monkeypatch, used_bytes=2 * (1 << 30)) + ok = ret.make_room_for( + db, str(rec), size=1, before_ts=100, + disk_pct=0, quota_gb=1, protect_ro=True, + ) + assert ok is False + assert _ids(db) == {"NEW.MP4"} # untouched + + +def test_make_room_evicts_until_under_then_true(env, monkeypatch): + rec, db = env + _clip(rec, db, basename="A.MP4", ts=100, size=1) + _clip(rec, db, basename="B.MP4", ts=200, size=1) + # Start over quota; each eviction frees ~1 GiB so the cached + # running total crosses below the 1 GiB cap. (Real files are + # tiny; we patch the freed-bytes return to GiB scale.) + _patch_quota(monkeypatch, used_bytes=2 * (1 << 30)) + monkeypatch.setattr( + "web.services.retention._delete_clip_files", + lambda row, recordings: 1 << 30, + ) + ok = ret.make_room_for( + db, str(rec), size=0, before_ts=300, + disk_pct=0, quota_gb=1, protect_ro=True, + ) + assert ok is True + # 2 GiB -> evict A -> 1 GiB (still >= cap) -> evict B -> 0 (< cap) -> stop + assert _ids(db) == set() + + +def test_make_room_honors_disk_pct_after_quota_satisfied(env, monkeypatch): + # Both rules set. Quota is satisfied from the start (used patched to 0), + # but disk_pct stays breached until BOTH clips are gone. Proves OR + # semantics: the loop keeps evicting for the disk rule even though the + # quota branch never fires. (Would fail if the loop exited as soon as + # quota was satisfied.) + import collections + rec, db = env + _clip(rec, db, basename="A.MP4", ts=100, size=1) + _clip(rec, db, basename="B.MP4", ts=200, size=1) + _patch_quota(monkeypatch, used_bytes=0) # quota never over + + DU = collections.namedtuple("DU", "total used free") + + def fake_du(path): + n = sum(1 for _ in rec.rglob("*.MP4")) + used = 100 if n > 0 else 0 # any file present -> 100% (>= 90) + return DU(total=100, used=used, free=100 - used) + + monkeypatch.setattr("web.services.retention.shutil.disk_usage", fake_du) + + ok = ret.make_room_for( + db, str(rec), size=0, before_ts=300, + disk_pct=90, quota_gb=1, protect_ro=True, + ) + assert ok is True + assert _ids(db) == set() # evicted until disk usage dropped under 90% diff --git a/tests/test_retention.py b/tests/test_retention.py index 1d1bea7..a3d1d15 100644 --- a/tests/test_retention.py +++ b/tests/test_retention.py @@ -233,7 +233,7 @@ def _patch_quota_scanner(monkeypatch, half_gib: int) -> None: ret._size_cache.clear() monkeypatch.setattr( ret, "_scan_dir_bytes", - lambda p: len(list(Path(p).rglob("*.MP4"))) * half_gib, + lambda p, exclude=frozenset(): len(list(Path(p).rglob("*.MP4"))) * half_gib, ) orig_del = ret._delete_clip_files def del_returning(*a, **kw): diff --git a/tests/test_settings_schema.py b/tests/test_settings_schema.py index 3039be4..4c6b5c0 100644 --- a/tests/test_settings_schema.py +++ b/tests/test_settings_schema.py @@ -228,3 +228,16 @@ def test_snapshot_exposes_disk_critical_pct(): env_file_path=f"{d}/v.env", recordings_dir=d) assert sp.get().disk_critical_pct == 95 + + +def test_import_path_round_trips_and_defaults_empty(): + from web.settings_schema import ( + DEFAULT_VALUES, EDITABLE_KEYS, validate_partial, + ) + # Editable + defaults to empty (meaning "/import"). + assert "IMPORT_PATH" in EDITABLE_KEYS + assert DEFAULT_VALUES["IMPORT_PATH"] == "" + # Round-trips through validate_partial, stripping whitespace. + out = validate_partial({"IMPORT_PATH": " /mnt/usb "}) + assert out["IMPORT_PATH"] == "/mnt/usb" + assert validate_partial({"IMPORT_PATH": " "})["IMPORT_PATH"] == "" diff --git a/tests/test_sync_worker_retention_loop.py b/tests/test_sync_worker_retention_loop.py index de5c6cb..aba3560 100644 --- a/tests/test_sync_worker_retention_loop.py +++ b/tests/test_sync_worker_retention_loop.py @@ -42,6 +42,7 @@ def _snap(**over): retention_disk_pct=0, retention_protect_ro=False, recordings_quota_gb=3000, + import_path="", ) base.update(over) return types.SimpleNamespace(**base) diff --git a/web/app.py b/web/app.py index 21d84ec..121ff7c 100644 --- a/web/app.py +++ b/web/app.py @@ -32,6 +32,7 @@ from .routers import mqtt as mqtt_router from .routers import setup as setup_router from .routers import storage as storage_router +from .routers import imports as imports_router from .services import retention as _ret_mod from .services import scanner from .services.exporter import ( @@ -256,6 +257,7 @@ def create_app() -> FastAPI: app.include_router(setup_router.router) app.include_router(mqtt_router.router) app.include_router(storage_router.router) + app.include_router(imports_router.router) # Static SPA — served at / with an explicit index.html fall-through # so the SPA's hash-router owns everything that isn't /api/*. diff --git a/web/routers/imports.py b/web/routers/imports.py new file mode 100644 index 0000000..a6d6846 --- /dev/null +++ b/web/routers/imports.py @@ -0,0 +1,158 @@ +"""Import endpoints — folder scan/ingest + per-file browser upload. + +Module is named ``imports`` (not ``import``, a Python keyword). All +routes require an authenticated session; mutating routes also require +CSRF, matching the other routers. +""" +from __future__ import annotations + +import asyncio +import logging +import os + +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel + +import viofosync_lib as vfs + +from ..auth import require_csrf, require_session +from ..services import importer +from ..services import retention as _retention + +log = logging.getLogger("viofosync.import") + +router = APIRouter( + prefix="/api/import", + tags=["import"], + dependencies=[Depends(require_session)], +) + + +class _PathBody(BaseModel): + path: str | None = None + + +def _db(request: Request): + return request.app.state.db + + +def _snap(request: Request): + return request.app.state.settings_provider.get() + + +def _resolve_root(snap, override: str | None) -> str: + return ( + override or snap.import_path + or os.path.join(snap.recordings, "import") + ).strip() + + +@router.post("/scan", dependencies=[Depends(require_csrf)]) +def scan(request: Request, body: _PathBody) -> dict: + snap = _snap(request) + root = _resolve_root(snap, body.path) + if not os.path.isdir(root): + raise HTTPException(400, f"not a readable directory: {root}") + man = importer.scan_source(root) + return { + "path": root, + "cross_volume": importer.is_cross_volume(root, snap.recordings), + "total_bytes": man.total_bytes, + "recognised": [importer.scan_item_dict(it) for it in man.items], + "skipped": man.skipped, + } + + +@router.post("/ingest", dependencies=[Depends(require_csrf)]) +async def ingest(request: Request, body: _PathBody) -> dict: + if getattr(request.app.state, "import_running", False): + raise HTTPException(409, "import already running") + snap = _snap(request) + root = _resolve_root(snap, body.path) + if not os.path.isdir(root): + raise HTTPException(400, f"not a readable directory: {root}") + + request.app.state.import_running = True + db = _db(request) + hub = request.app.state.hub + loop = asyncio.get_running_loop() + + def _work(): + try: + importer.run_folder_ingest(db, snap, hub, loop, root=root) + except Exception: # pragma: no cover — never wedge the flag + log.exception("folder ingest failed") + finally: + request.app.state.import_running = False + + loop.run_in_executor(None, _work) + return {"ok": True, "started": True} + + +@router.post("/upload", dependencies=[Depends(require_csrf)]) +async def upload(request: Request) -> dict: + snap = _snap(request) + db = _db(request) + rel = request.headers.get("X-Import-Path", "") + name = os.path.basename(rel.replace("\\", "/")) + try: + size = int(request.headers.get("X-Import-Size") + or request.headers.get("Content-Length") or 0) + except ValueError: + size = 0 + + m = vfs.downloaded_filename_re.match(name) + if not m: + return {"status": "not_recognised", "filename": name} + + # Destination derives ONLY from the parsed basename — the client's + # relative path is used solely for RO detection, never the write path. + item = importer.scan_item_from_match( + m, name, source_rel_path=rel, size=size, src_path="", + ) + dest = importer.dest_for(snap, item) + if os.path.exists(dest): + return {"status": "already_present", "filename": name} + + # Evict to fit BEFORE writing bytes (size known from the header). + if not _retention.make_room_for( + db, snap.recordings, size=item.size_bytes, before_ts=item.timestamp, + disk_pct=snap.retention_disk_pct, quota_gb=snap.recordings_quota_gb, + protect_ro=snap.retention_protect_ro, + exclude=_retention.import_exclude_set(snap.recordings, snap.import_path), + ): + log.warning("upload rejected (over quota, older than retained set): %s", name) + return {"status": "over_quota_older", "filename": name} + + staging = os.path.join(snap.recordings, importer.STAGING_DIRNAME) + os.makedirs(staging, exist_ok=True) + tmp = os.path.join(staging, name) + written = 0 + try: + with open(tmp, "wb") as f: + async for chunk in request.stream(): + f.write(chunk) + written += len(chunk) + except Exception as e: # pragma: no cover — client abort / disk error + _silent_remove(tmp) + log.warning("upload stream failed for %s: %s", name, e) + return {"status": "error", "filename": name, "detail": str(e)} + + if size and written != size: + _silent_remove(tmp) + log.warning("upload size mismatch for %s: got %d, expected %d", name, written, size) + return {"status": "error", "filename": name, "detail": "size mismatch"} + + item.size_bytes = written + item.src_path = tmp + res = await asyncio.to_thread( + importer.ingest_clip, db, snap, item, cross_volume=False, staged=True, + ) + return importer.clip_result_dict(res) + + +def _silent_remove(path: str) -> None: + try: + os.remove(path) + except OSError: + pass diff --git a/web/routers/settings.py b/web/routers/settings.py index 2a67978..fcb7949 100644 --- a/web/routers/settings.py +++ b/web/routers/settings.py @@ -33,6 +33,7 @@ def _editable_values(snap) -> dict[str, Any]: return { "ADDRESS": snap.address or "", "ADDRESS_FALLBACK": snap.address_fallback or "", + "IMPORT_PATH": snap.import_path, "GROUPING": snap.grouping, "HTML": snap.use_html_listing, "GPS_EXTRACT": snap.gps_extract, diff --git a/web/services/importer.py b/web/services/importer.py new file mode 100644 index 0000000..747c9a6 --- /dev/null +++ b/web/services/importer.py @@ -0,0 +1,319 @@ +"""Local import / ingest core. + +Feeds locally-available Viofo clips (browser upload, drop folder, or +an external/USB mount) into the archive, reusing the same filename +patterns, path layout, GPX, indexing, and retention as Wi-Fi sync. + +Pure-ish: ``scan_source`` and ``ingest_clip`` do filesystem + DB work +but no asyncio; ``run_folder_ingest`` is the batch driver invoked on a +worker thread, broadcasting progress via the hub. +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +import logging +import os +import re +import shutil +import time +from dataclasses import asdict, dataclass, field + +import viofosync_lib as vfs + +from ..db import Database +from . import retention as _retention +from . import scanner + +log = logging.getLogger("viofosync.importer") + +STAGING_DIRNAME = ".import_tmp" + + +@dataclass +class ClipResult: + filename: str + status: str # imported|already_present|not_recognised + # |over_quota_older|error + detail: str = "" + size_bytes: int = 0 + event_type: str = "normal" + + +@dataclass +class ScanItem: + src_path: str + source_rel_path: str + basename: str + timestamp: int + camera: str + sequence: int + event_type: str + size_bytes: int + + +@dataclass +class Manifest: + items: list[ScanItem] = field(default_factory=list) # newest-first + skipped: list[dict] = field(default_factory=list) # [{"name","reason"}] + total_bytes: int = 0 + + +def _is_ro(source_rel_path: str) -> bool: + norm = "/" + source_rel_path.replace("\\", "/").strip("/").upper() + return "/RO/" in norm + + +def classify_event_type(camera_field: str, source_rel_path: str) -> str: + if _is_ro(source_rel_path): + return "ro" + if camera_field.upper().startswith("P"): + return "parking" + return "normal" + + +def scan_item_from_match( + m: re.Match[str], name: str, *, source_rel_path: str, size: int, src_path: str, +) -> ScanItem: + ts = int(_dt.datetime( + int(m.group("year")), int(m.group("month")), int(m.group("day")), + int(m.group("hour")), int(m.group("minute")), int(m.group("second")), + ).timestamp()) + cam = m.group("camera") + return ScanItem( + src_path=src_path, source_rel_path=source_rel_path, basename=name, + timestamp=ts, camera=cam.upper(), sequence=int(m.group("sequence")), + event_type=classify_event_type(cam, source_rel_path), size_bytes=size, + ) + + +def scan_source(root: str) -> Manifest: + """Recurse ``root``, returning recognised Viofo clips (newest-first) + and a list of skipped files with reasons.""" + man = Manifest() + for dirpath, _dirs, files in os.walk(root): + for name in files: + full = os.path.join(dirpath, name) + m = vfs.downloaded_filename_re.match(name) + if not m: + man.skipped.append({"name": name, "reason": "not_recognised"}) + continue + try: + size = os.path.getsize(full) + except OSError: + size = 0 + rel = os.path.relpath(full, root) + try: + item = scan_item_from_match( + m, name, source_rel_path=rel, size=size, src_path=full, + ) + except ValueError: + man.skipped.append({"name": name, "reason": "bad_timestamp"}) + continue + man.items.append(item) + man.total_bytes += size + man.items.sort(key=lambda it: it.timestamp, reverse=True) + return man + + +def scan_item_dict(it: ScanItem) -> dict: + return asdict(it) + + +def clip_result_dict(res: ClipResult) -> dict: + return asdict(res) + + +def dest_for(snap, item: ScanItem) -> str: + group = vfs.get_group_name( + _dt.datetime.fromtimestamp(item.timestamp), snap.grouping, + ) + return vfs.get_filepath(snap.recordings, group or "", item.basename) + + +def _origin_source_dir(item: ScanItem) -> str: + # Invariant: contains "/RO/" iff the clip is locked, so scanner.scan + # re-derives event_type='ro' on every future rescan. + return "/import/RO/" if item.event_type == "ro" else "/import/" + + +def _record_origin(db: Database, item: ScanItem) -> None: + now = int(time.time()) + with db.write() as c: + c.execute( + "INSERT OR IGNORE INTO download_queue " + "(filename, source_dir, remote_size, recorded_at, camera, " + " event_type, state, priority, enqueued_at, finished_at, manual) " + "VALUES (?, ?, ?, ?, ?, ?, 'done', 0, ?, ?, 1)", + (item.basename, _origin_source_dir(item), item.size_bytes, + item.timestamp, item.camera, item.event_type, now, now), + ) + + +def is_cross_volume(root: str, recordings: str) -> bool: + try: + return os.stat(root).st_dev != os.stat(recordings).st_dev + except OSError: + return True # safe default: copy, never a destructive move + + +def ingest_clip( + db: Database, snap, item: ScanItem, *, + cross_volume: bool, staged: bool = False, +) -> ClipResult: + """Place one clip into the archive. ``staged`` means ``item.src_path`` + is already in ``.import_tmp`` and make-room was done by the caller + (the upload path); we then go straight to the final rename.""" + recordings = snap.recordings + dest = dest_for(snap, item) + if os.path.exists(dest): + return ClipResult(item.basename, "already_present", + size_bytes=item.size_bytes, event_type=item.event_type) + + if not staged: + ok = _retention.make_room_for( + db, recordings, size=item.size_bytes, before_ts=item.timestamp, + disk_pct=snap.retention_disk_pct, quota_gb=snap.recordings_quota_gb, + protect_ro=snap.retention_protect_ro, + exclude=_retention.import_exclude_set(recordings, snap.import_path), + ) + if not ok: + return ClipResult(item.basename, "over_quota_older", + size_bytes=item.size_bytes, event_type=item.event_type) + + staging = os.path.join(recordings, STAGING_DIRNAME) + tmp = item.src_path if staged else os.path.join(staging, item.basename) + if not staged: + os.makedirs(staging, exist_ok=True) + try: + if cross_volume: + shutil.copy2(item.src_path, tmp) + if os.path.getsize(tmp) != item.size_bytes: + os.remove(tmp) + log.warning( + "import size mismatch for %s after copy", item.basename + ) + return ClipResult(item.basename, "error", + detail="size mismatch after copy", + size_bytes=item.size_bytes, + event_type=item.event_type) + else: + os.replace(item.src_path, tmp) # same-volume move into staging + except OSError as e: + log.warning("import staging failed for %s: %s", item.basename, e) + return ClipResult(item.basename, "error", detail=str(e), + size_bytes=item.size_bytes, + event_type=item.event_type) + + try: + os.makedirs(os.path.dirname(dest), exist_ok=True) + os.replace(tmp, dest) + except OSError as e: + # Don't strand the staged file. A same-volume move has already + # consumed the source, so restore it (no data loss); cross-volume + # keeps an intact original and upload (staged) will be retried, so + # the temp is safe to drop there. + if not staged and not cross_volume: + try: + os.replace(tmp, item.src_path) + except OSError as restore_err: + log.error( + "import: failed to restore source for %s after a failed " + "placement; staged file %s may be lost: %s", + item.basename, tmp, restore_err, + ) + else: + try: + os.remove(tmp) + except OSError: + pass + log.warning("import placement failed for %s: %s", item.basename, e) + return ClipResult(item.basename, "error", detail=str(e), + size_bytes=item.size_bytes, event_type=item.event_type) + + if snap.gps_extract: + try: + vfs.extract_gps_data(dest) + except Exception as e: # clips without a GPS lock — non-fatal + log.info("gpx extract failed for %s: %s", item.basename, e) + + _record_origin(db, item) + return ClipResult(item.basename, "imported", + size_bytes=item.size_bytes, event_type=item.event_type) + + +def _clean_staging(recordings: str) -> None: + staging = os.path.join(recordings, STAGING_DIRNAME) + if not os.path.isdir(staging): + return + for name in os.listdir(staging): + try: + os.remove(os.path.join(staging, name)) + except OSError: # pragma: no cover — best-effort + pass + + +def _broadcast(hub, loop, event: dict) -> None: + if hub is None: + return + hub.schedule_broadcast(loop, event) + + +_SUMMARY_KEYS = ( + "imported", "already_present", + "over_quota_older", "errors", +) + + +def run_folder_ingest(db: Database, snap, hub, loop, *, root: str) -> dict: + """Ingest every recognised clip under ``root`` (newest-first), + then run the post-sync pipeline. Runs on a worker thread.""" + recordings = snap.recordings + _clean_staging(recordings) # clear debris from any prior aborted run + man = scan_source(root) + cross = is_cross_volume(root, recordings) + + summary = {k: 0 for k in _SUMMARY_KEYS} + summary["not_recognised"] = len(man.skipped) # from the manifest, not per-clip + summary["bytes_imported"] = 0 + total = len(man.items) + _broadcast(hub, loop, {"type": "import_started", "total": total}) + + for i, item in enumerate(man.items, 1): + res = ingest_clip(db, snap, item, cross_volume=cross) + if res.status in summary: + summary[res.status] += 1 + else: # 'error' + summary["errors"] += 1 + if res.status == "imported": + summary["bytes_imported"] += res.size_bytes + _broadcast(hub, loop, { + "type": "import_progress", "done": i, "total": total, + "filename": res.filename, "result": res.status, + }) + + # Reuse the post-sync pipeline. scanner.scan handles its own + # broadcast + threadsafe scheduling. + scanner.scan(db, recordings, snap.grouping, hub, loop) + if loop is not None: + asyncio.run_coroutine_threadsafe( + scanner.sweep_missing_thumbs(db, recordings), loop, + ) + _retention.sweep( + db, recordings, + max_days=snap.retention_max_days, + disk_pct=snap.retention_disk_pct, + protect_ro=snap.retention_protect_ro, + quota_gb=snap.recordings_quota_gb, + exclude=_retention.import_exclude_set(recordings, snap.import_path), + ) + log.info( + "import complete: %d imported, %d already_present, %d not_recognised, " + "%d over_quota_older, %d error(s)", + summary["imported"], summary["already_present"], + summary["not_recognised"], summary["over_quota_older"], + summary["errors"], + ) + _broadcast(hub, loop, {"type": "import_done", **summary}) + _clean_staging(recordings) + return summary diff --git a/web/services/retention.py b/web/services/retention.py index 9092a89..79d4fa6 100644 --- a/web/services/retention.py +++ b/web/services/retention.py @@ -97,6 +97,7 @@ def sweep( protect_ro: bool, quota_gb: int = 0, sink=None, + exclude: frozenset[str] = frozenset(), _now: Optional[int] = None, ) -> dict: """Run the retention pass. Returns a summary dict. @@ -158,6 +159,7 @@ def sweep( quota_gb=quota_gb, protect_ro=protect_ro, sink=sink, + exclude=exclude, ) bytes_freed += freed_2 protected += protected_2 @@ -184,12 +186,14 @@ def sweep( # quota mode: deletes subtract from the cached total so the inner # bail-out check in _disk_pressure_pass doesn't trigger a fresh tree # walk after every file. -_size_cache: dict[str, tuple[float, int]] = {} +_size_cache: dict[tuple[str, frozenset[str]], tuple[float, int]] = {} -def _scan_dir_bytes(path: str) -> int: +def _scan_dir_bytes(path: str, exclude: frozenset[str] = frozenset()) -> int: """Sum of file sizes in ``path``, recursing without crossing mount - points. Used by quota mode in place of ``shutil.disk_usage`` when + points. Directories whose absolute path is in ``exclude`` are + skipped wholesale — used to keep import staging dirs off the quota + books. Used by quota mode in place of ``shutil.disk_usage`` when the OS-level free-space figure doesn't reflect the actual quota (e.g. Synology shared folder, ZFS dataset, NFS share).""" try: @@ -210,6 +214,8 @@ def _scan_dir_bytes(path: str) -> int: if st.st_dev != root_dev: continue if entry.is_dir(follow_symlinks=False): + if os.path.abspath(entry.path) in exclude: + continue stack.append(entry.path) elif entry.is_file(follow_symlinks=False): total += st.st_size @@ -218,23 +224,32 @@ def _scan_dir_bytes(path: str) -> int: return total -def _cached_used_bytes(path: str, *, refresh: bool = False) -> int: +def _cached_used_bytes( + path: str, *, refresh: bool = False, exclude: frozenset[str] = frozenset() +) -> int: now = _time_mod.monotonic() - cached = _size_cache.get(path) + key = (path, exclude) + cached = _size_cache.get(key) if not refresh and cached and (now - cached[0]) < _SIZE_CACHE_TTL: return cached[1] - used = _scan_dir_bytes(path) - _size_cache[path] = (now, used) + used = _scan_dir_bytes(path, exclude=exclude) + _size_cache[key] = (now, used) return used -def _cache_subtract(path: str, freed: int) -> None: - cached = _size_cache.get(path) +def _cache_subtract( + path: str, freed: int, *, exclude: frozenset[str] = frozenset() +) -> None: + key = (path, exclude) + cached = _size_cache.get(key) if cached is not None and freed > 0: - _size_cache[path] = (cached[0], max(0, cached[1] - freed)) + _size_cache[key] = (cached[0], max(0, cached[1] - freed)) -def disk_used_pct(recordings: str, quota_gb: int = 0) -> Optional[float]: +def disk_used_pct( + recordings: str, quota_gb: int = 0, + exclude: frozenset[str] = frozenset(), +) -> Optional[float]: """Public helper for consumers (MQTT, status APIs) that want the same used-% the retention sweep evaluates against. @@ -255,7 +270,7 @@ def disk_used_pct(recordings: str, quota_gb: int = 0) -> Optional[float]: a perpetual error. """ if quota_gb > 0: - used = _cached_used_bytes(recordings) + used = _cached_used_bytes(recordings, exclude=exclude) limit = quota_gb * (1 << 30) if limit <= 0: return None @@ -290,24 +305,30 @@ def _pct_exceeded(recordings: str, disk_pct: int) -> bool: return (du.used / du.total * 100.0) >= disk_pct -def _quota_exceeded(recordings: str, quota_gb: int, *, refresh: bool = False) -> bool: +def _quota_exceeded( + recordings: str, quota_gb: int, *, refresh: bool = False, + exclude: frozenset[str] = frozenset(), +) -> bool: """Absolute-quota rule. ``quota_gb == 0`` disables it. Reads from the cached size-walk (decremented in-place by each delete) so the inner sweep loop doesn't pay for a tree walk per file.""" if quota_gb <= 0: return False - return _cached_used_bytes(recordings, refresh=refresh) >= quota_gb * (1 << 30) + return _cached_used_bytes( + recordings, refresh=refresh, exclude=exclude + ) >= quota_gb * (1 << 30) def _over_threshold( - recordings: str, *, disk_pct: int, quota_gb: int, refresh: bool = False + recordings: str, *, disk_pct: int, quota_gb: int, refresh: bool = False, + exclude: frozenset[str] = frozenset(), ) -> bool: """True if EITHER rule is currently breached. Independent triggers — set the percentage to bound the underlying filesystem, set the quota to bound bytes-under-recordings, or set both.""" return ( _pct_exceeded(recordings, disk_pct) - or _quota_exceeded(recordings, quota_gb, refresh=refresh) + or _quota_exceeded(recordings, quota_gb, refresh=refresh, exclude=exclude) ) @@ -319,6 +340,7 @@ def _disk_pressure_pass( quota_gb: int, protect_ro: bool, sink, + exclude: frozenset[str] = frozenset(), ) -> tuple[int, int, int]: """Delete oldest clips first until both pressure rules are satisfied or no more eligible candidates remain. @@ -348,6 +370,7 @@ def _disk_pressure_pass( bytes_freed = 0 while _over_threshold( recordings, disk_pct=disk_pct, quota_gb=quota_gb, refresh=True, + exclude=exclude, ): where = "" if protect_ro: @@ -365,19 +388,21 @@ def _disk_pressure_pass( break for row in rows: freed = _delete_clip_files(row, recordings) - _cache_subtract(recordings, freed) + _cache_subtract(recordings, freed, exclude=exclude) bytes_freed += freed _delete_index_row(db, row["id"]) deleted += 1 _broadcast(sink, row["basename"], "disk") if not _over_threshold( recordings, disk_pct=disk_pct, quota_gb=quota_gb, + exclude=exclude, ): return deleted, bytes_freed, 0 protected = 0 if protect_ro and _over_threshold( recordings, disk_pct=disk_pct, quota_gb=quota_gb, refresh=True, + exclude=exclude, ): with db.conn() as c: protected = c.execute( @@ -385,3 +410,77 @@ def _disk_pressure_pass( "WHERE COALESCE(event_type, '') = 'ro'" ).fetchone()["n"] return deleted, bytes_freed, protected + + +def make_room_for( + db: Database, recordings: str, *, + size: int, before_ts: int, + disk_pct: int, quota_gb: int, protect_ro: bool, + exclude: frozenset[str] = frozenset(), +) -> bool: + """Ensure ``size`` more bytes will fit under the active rules, + by deleting the OLDEST clip whose timestamp < ``before_ts`` + (skipping ``ro`` when ``protect_ro``). Returns False when no + evictable clip older than ``before_ts`` remains while still + over threshold — the caller then skips that clip. Never deletes + a clip newer than or equal to the one being imported. + + With neither rule set, returns True (rely on the filesystem). + + The quota total is walked ONCE up front and decremented by the + bytes each delete frees, so a multi-eviction call does not + re-walk the tree per file (matching the size-cache philosophy + used elsewhere in this module). + """ + if disk_pct <= 0 and quota_gb <= 0: + return True + + quota_bytes = quota_gb * (1 << 30) if quota_gb > 0 else None + used = _scan_dir_bytes(recordings, exclude=exclude) if quota_bytes else 0 + + def _over() -> bool: + if quota_bytes is not None and used + size >= quota_bytes: + return True + if disk_pct > 0: + try: + du = shutil.disk_usage(recordings) + except OSError: + return False + if du.total > 0 and ((du.used + size) / du.total * 100.0) >= disk_pct: + return True + return False + + where = "WHERE timestamp < ?" + params: list = [before_ts] + if protect_ro: + where += " AND COALESCE(event_type, '') != 'ro'" + + while _over(): + with db.conn() as c: + row = c.execute( + f"SELECT id, path, basename FROM clip_index {where} " + f"ORDER BY timestamp ASC LIMIT 1", + params, + ).fetchone() + if row is None: + return False + freed = _delete_clip_files(dict(row), recordings) + _delete_index_row(db, row["id"]) + used = max(0, used - freed) + return True + + +def import_exclude_set(recordings: str, import_path: str = "") -> frozenset[str]: + """Absolute dirs to keep off the quota walk during import: the + ``.import_tmp`` staging dir, and the resolved import drop folder + when it lives inside the recordings tree (a same-volume external + mount is a different st_dev and is never counted anyway).""" + rec_abs = os.path.abspath(recordings) + out = {os.path.join(rec_abs, ".import_tmp")} + resolved = os.path.abspath(import_path or os.path.join(recordings, "import")) + try: + if resolved != rec_abs and os.path.commonpath([resolved, rec_abs]) == rec_abs: + out.add(resolved) + except ValueError: # different drives (Windows) — not under recordings + pass + return frozenset(out) diff --git a/web/services/sync_worker.py b/web/services/sync_worker.py index ea78167..ed4fa12 100644 --- a/web/services/sync_worker.py +++ b/web/services/sync_worker.py @@ -553,6 +553,9 @@ async def _run_retention_sweep(self) -> None: protect_ro=snap.retention_protect_ro, quota_gb=snap.recordings_quota_gb, sink=sink, + exclude=_retention.import_exclude_set( + snap.recordings, snap.import_path + ), ) await self._emit_disk_pct() @@ -704,6 +707,9 @@ async def _cycle(self) -> bool: protect_ro=snap.retention_protect_ro, quota_gb=snap.recordings_quota_gb, sink=sink, + exclude=_retention.import_exclude_set( + snap.recordings, snap.import_path + ), ) except Exception: # pragma: no cover — non-fatal log.exception("post-cycle scan/thumb sweep failed") diff --git a/web/settings.py b/web/settings.py index b457506..8d7e745 100644 --- a/web/settings.py +++ b/web/settings.py @@ -43,6 +43,7 @@ class Snapshot: address: str | None address_fallback: str | None recordings: str + import_path: str grouping: str use_html_listing: bool gps_extract: bool @@ -226,6 +227,7 @@ def _make_snapshot(self, data: dict) -> Snapshot: address=m.ADDRESS, address_fallback=m.ADDRESS_FALLBACK, recordings=self._recordings, + import_path=m.IMPORT_PATH, grouping=m.GROUPING, use_html_listing=m.HTML, gps_extract=m.GPS_EXTRACT, diff --git a/web/settings_schema.py b/web/settings_schema.py index 7633cfa..733e530 100644 --- a/web/settings_schema.py +++ b/web/settings_schema.py @@ -29,6 +29,7 @@ class SettingsModel(BaseModel): ADDRESS: str | None = None ADDRESS_FALLBACK: str | None = None + IMPORT_PATH: str = "" WEB_PASSWORD_HASH: str = "" SESSION_SECRET: str = Field(default_factory=lambda: secrets.token_hex(32)) @@ -81,6 +82,11 @@ class SettingsModel(BaseModel): MQTT_DISCOVERY_ENABLED: bool = True MQTT_QOS: Literal[0, 1, 2] = 1 + @field_validator("IMPORT_PATH") + @classmethod + def _validate_import_path(cls, v: str) -> str: + return v.strip() + @field_validator("ADDRESS", "ADDRESS_FALLBACK") @classmethod def _validate_address(cls, v: str | None) -> str | None: @@ -149,7 +155,7 @@ def _validate_disk_critical(self): # Public taxonomy used by the API + UI. EDITABLE_KEYS = { - "ADDRESS", "ADDRESS_FALLBACK", "GROUPING", "HTML", "GPS_EXTRACT", + "ADDRESS", "ADDRESS_FALLBACK", "IMPORT_PATH", "GROUPING", "HTML", "GPS_EXTRACT", "DELETE_AFTER_DOWNLOAD", "TIMEOUT", "DOWNLOAD_ATTEMPTS", "MAX_DOWNLOAD_ATTEMPTS", "SYNC_INTERVAL", "ENABLE_SCHEDULED_SYNC", "WEB_HOST", "WEB_PORT", "EXPORT_ENCODER", diff --git a/web/static/app.js b/web/static/app.js index 6417a59..d46509d 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -2138,6 +2138,11 @@ function handleEvent(ev) { refreshExportJobs(); } break; + case "import_started": + case "import_progress": + case "import_done": + if (window.__importOnEvent) window.__importOnEvent(ev); + break; } } @@ -2897,3 +2902,218 @@ window.addEventListener("hashchange", () => { showLogin(); } })(); + +// ---- Import modal ------------------------------------------------------- +(function importModal() { + const modal = document.getElementById("import-modal"); + if (!modal) return; + const $ = (id) => document.getElementById(id); + const show = (el) => el && el.classList.remove("hidden"); + const hide = (el) => el && el.classList.add("hidden"); + const csrfH = () => (state.csrf ? { "x-csrf-token": state.csrf } : {}); + + $("import-btn").addEventListener("click", () => { + hide($("import-summary")); hide($("import-progress")); + show(modal); + }); + $("import-close").addEventListener("click", () => hide(modal)); + + modal.querySelectorAll(".import-tab").forEach((tab) => { + tab.addEventListener("click", () => { + modal.querySelectorAll(".import-tab").forEach((t) => t.classList.remove("active")); + tab.classList.add("active"); + modal.querySelectorAll(".tab-pane").forEach((p) => hide(p)); + show(modal.querySelector(`[data-pane="${tab.dataset.tab}"]`)); + }); + }); + + const RE = /^\d{4}_\d{4}_\d{6}_\d+.+\.MP4$/i; + const tsOf = (n) => { + const m = n.match(/^(\d{4})_(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})/); + return m ? Number(m.slice(1).join("")) : 0; + }; + + // --- Upload tab --- + // Each entry is { file, path } so the folder picker (relative path via + // webkitRelativePath) and drag-dropped folders (path from the + // FileSystemEntry walk) share one shape: the path drives RO detection + // server-side, the basename drives the write location. + let picked = []; + const dz = modal.querySelector(".dropzone"); + const dzTitle = modal.querySelector(".dropzone-title"); + + function applySelection(items) { + picked = items + .filter((it) => RE.test(it.file.name)) + .sort((a, b) => tsOf(b.file.name) - tsOf(a.file.name)); // newest-first + const skipped = items.length - picked.length; + dz.classList.toggle("has-files", picked.length > 0); + dzTitle.textContent = picked.length + ? `${picked.length} clip${picked.length === 1 ? "" : "s"} ready` + : "Drag a folder here, or click to browse"; + $("import-upload-manifest").textContent = items.length + ? (picked.length + ? `${picked.length} recognised${skipped ? `, ${skipped} skipped` : ""} — newest first.` + : "No Viofo clips found in that folder.") + : ""; + $("import-upload-go").disabled = picked.length === 0; + } + + $("import-files").addEventListener("change", (e) => { + applySelection( + [...e.target.files].map((f) => ({ file: f, path: f.webkitRelativePath || f.name })), + ); + }); + + // Drag-and-drop a folder onto the dropzone. The browser default is to + // navigate to the dropped item, so every handler must preventDefault. + const readEntries = (reader) => + new Promise((res, rej) => reader.readEntries(res, rej)); + async function walk(entry, prefix, out) { + const here = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isFile) { + out.push({ file: await new Promise((res, rej) => entry.file(res, rej)), path: here }); + } else if (entry.isDirectory) { + const reader = entry.createReader(); + let batch; + do { + batch = await readEntries(reader); + for (const child of batch) await walk(child, here, out); + } while (batch.length); + } + } + + const stop = (e) => { e.preventDefault(); e.stopPropagation(); }; + ["dragenter", "dragover"].forEach((ev) => + dz.addEventListener(ev, (e) => { stop(e); dz.classList.add("dragging"); })); + dz.addEventListener("dragleave", (e) => { + if (!dz.contains(e.relatedTarget)) dz.classList.remove("dragging"); + }); + dz.addEventListener("drop", async (e) => { + stop(e); + dz.classList.remove("dragging"); + const dt = e.dataTransfer; + // webkitGetAsEntry must be read synchronously, before any await. + const entries = dt.items + ? [...dt.items] + .filter((i) => i.kind === "file") + .map((i) => (i.webkitGetAsEntry ? i.webkitGetAsEntry() : null)) + .filter(Boolean) + : []; + const plain = entries.length ? [] : [...dt.files]; + const out = []; + for (const entry of entries) await walk(entry, "", out); + for (const f of plain) out.push({ file: f, path: f.name }); + applySelection(out); + }); + + // Stop a near-miss drop (onto the card or backdrop) from navigating the + // page away while the modal is open; real drops on the zone are handled + // above. Only active while the modal is open, so app-wide DnD is untouched. + const guardStrayDrop = (e) => { if (!modal.classList.contains("hidden")) e.preventDefault(); }; + window.addEventListener("dragover", guardStrayDrop); + window.addEventListener("drop", guardStrayDrop); + + $("import-upload-go").addEventListener("click", async () => { + show($("import-progress")); hide($("import-summary")); + const tally = {}; + for (let i = 0; i < picked.length; i++) { + const { file, path } = picked[i]; + $("import-status").textContent = `Uploading ${file.name} (${i + 1}/${picked.length})`; + $("import-bar").style.width = `${(i / picked.length) * 100}%`; + let res; + try { + const r = await fetch("/api/import/upload", { + method: "POST", + credentials: "same-origin", + headers: { + ...csrfH(), + "X-Import-Path": path, + "X-Import-Size": String(file.size), + "Content-Type": "application/octet-stream", + }, + body: file, + }); + if (!r.ok) { + let detail = r.status; + try { detail = (await r.json()).detail || r.status; } catch (_) {} + res = { status: "error", detail: String(detail) }; + } else { + res = await r.json(); + } + } catch (err) { + res = { status: "error", detail: String(err) }; + } + const key = res.status === "error" ? "errors" : res.status; + tally[key] = (tally[key] || 0) + 1; + } + $("import-bar").style.width = "100%"; + await fetch("/api/archive/rescan", { method: "POST", credentials: "same-origin", headers: csrfH() }); + renderSummary(tally); + }); + + // --- Folder tab --- + $("import-folder-scan").addEventListener("click", async () => { + const path = $("import-folder-path").value.trim() || null; + const r = await fetch("/api/import/scan", { + method: "POST", + credentials: "same-origin", + headers: { ...csrfH(), "Content-Type": "application/json" }, + body: JSON.stringify({ path }), + }); + if (!r.ok) { + $("import-folder-manifest").textContent = `Error: ${(await r.json()).detail || r.status}`; + hide($("import-folder-go")); + return; + } + const m = await r.json(); + $("import-folder-manifest").textContent = + `${m.recognised.length} clip(s), ${m.skipped.length} skipped, ` + + `${(m.total_bytes / 1e9).toFixed(2)} GB${m.cross_volume ? " (external — copy)" : ""}.`; + $("import-folder-go").dataset.path = path || ""; + show($("import-folder-go")); + }); + + $("import-folder-go").addEventListener("click", async (e) => { + const path = e.target.dataset.path || null; + show($("import-progress")); hide($("import-summary")); + $("import-status").textContent = "Starting…"; + const r = await fetch("/api/import/ingest", { + method: "POST", + credentials: "same-origin", + headers: { ...csrfH(), "Content-Type": "application/json" }, + body: JSON.stringify({ path }), + }); + if (!r.ok) { + hide($("import-progress")); + let detail = r.status; + try { detail = (await r.json()).detail || r.status; } catch (_) {} + $("import-status").textContent = `Error: ${detail}`; + show($("import-status")); + } + // Progress arrives over the WebSocket (import_* events). + }); + + function renderSummary(t) { + hide($("import-progress")); + const el = $("import-summary"); + el.textContent = + `Imported ${t.imported || 0}, duplicate ${t.already_present || 0}, ` + + `skipped(quota) ${t.over_quota_older || 0}, ` + + `unrecognised ${t.not_recognised || 0}, errors ${t.errors || 0}.`; + show(el); + if (!document.getElementById("view-archive").hidden) loadDays(); + } + + // Fold folder-mode WS events into the same UI. + window.__importOnEvent = (ev) => { + if (ev.type === "import_started") { + show($("import-progress")); $("import-bar").style.width = "0%"; + } else if (ev.type === "import_progress") { + $("import-bar").style.width = `${(ev.done / Math.max(ev.total, 1)) * 100}%`; + $("import-status").textContent = `${ev.filename} (${ev.done}/${ev.total})`; + } else if (ev.type === "import_done") { + renderSummary(ev); + } + }; +})(); diff --git a/web/static/index.html b/web/static/index.html index ac8435c..d2c3e72 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -133,6 +133,16 @@

Archive

+ + diff --git a/web/static/styles.css b/web/static/styles.css index 60625d4..07f1a7e 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -1698,3 +1698,163 @@ main { .storage-usage-mode { margin: 0.5rem 0 0; } + +/* ---- Import recordings modal ---- */ +#import-modal { + position: fixed; inset: 0; z-index: 1500; + display: flex; align-items: center; justify-content: center; padding: 24px; + background: oklch(0.10 0.008 250 / 0.62); + -webkit-backdrop-filter: blur(6px); backdrop-filter: blur(6px); + animation: import-fade 160ms ease; +} +#import-modal.hidden { display: none; } +@keyframes import-fade { from { opacity: 0; } to { opacity: 1; } } +@keyframes import-rise { + from { opacity: 0; transform: translateY(10px) scale(0.985); } + to { opacity: 1; transform: none; } +} + +#import-modal .modal-card { + width: min(520px, 100%); max-height: 88vh; overflow: auto; + background: var(--panel); color: var(--text); + border: 1px solid var(--border); border-radius: 14px; + box-shadow: 0 24px 60px oklch(0 0 0 / 0.55); + padding: 22px 24px 24px; + animation: import-rise 220ms cubic-bezier(0.22, 1, 0.36, 1); +} + +#import-modal .modal-head { + display: flex; align-items: center; justify-content: space-between; gap: 12px; + padding-bottom: 14px; margin-bottom: 16px; + border-bottom: 1px solid var(--border); +} +#import-modal .modal-head h2 { + margin: 0; font-size: 1.15rem; letter-spacing: -0.01em; +} +#import-modal .modal-x { + display: flex; align-items: center; justify-content: center; + width: 32px; height: 32px; padding: 0; flex-shrink: 0; + border: 0; border-radius: 8px; background: transparent; color: var(--muted); + transition: background 140ms ease, color 140ms ease; +} +#import-modal .modal-x:hover { background: var(--panel-3); color: var(--text); } + +/* Tabs — underline, muted until active */ +#import-modal .tabs { + display: flex; gap: 4px; margin: 0 0 18px; border-bottom: 1px solid var(--border); +} +#import-modal .import-tab { + padding: 8px 14px; margin-bottom: -1px; + border: 0; background: transparent; cursor: pointer; + color: var(--muted); font-size: 0.92rem; font-weight: 500; + border-bottom: 2px solid transparent; + transition: color 140ms ease, border-color 140ms ease; +} +#import-modal .import-tab:hover { color: var(--text); } +#import-modal .import-tab.active { color: var(--text); border-bottom-color: var(--accent); } + +#import-modal .tab-pane.hidden { display: none; } +#import-modal .pane-lead { + margin: 0 0 14px; color: var(--muted); font-size: 0.9rem; line-height: 1.5; +} +#import-modal code { + font-family: "SF Mono", "Menlo", monospace; font-size: 0.85em; + background: var(--panel-2); border: 1px solid var(--border); + border-radius: 4px; padding: 1px 5px; +} + +/* Upload dropzone (native input visually hidden, label is the target) */ +#import-modal #import-files { + position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; + overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; border: 0; +} +#import-modal .dropzone { + display: flex; flex-direction: column; align-items: center; gap: 8px; + text-align: center; padding: 30px 20px; + border: 1.5px dashed var(--border-hover); border-radius: 12px; + background: var(--panel-2); cursor: pointer; + transition: border-color 160ms ease, background 160ms ease; +} +#import-modal .dropzone:hover { border-color: var(--accent); background: var(--accent-08); } +#import-modal #import-files:focus-visible + .dropzone { + border-color: var(--accent); box-shadow: 0 0 0 3px var(--accent-35); +} +#import-modal .dropzone.has-files { + border-style: solid; border-color: var(--accent); background: var(--accent-08); +} +#import-modal .dropzone.dragging { + border-style: solid; border-color: var(--accent); background: var(--accent-18); +} +#import-modal .dropzone-icon { color: var(--accent); } +#import-modal .dropzone-title { font-weight: 600; font-size: 0.98rem; } +#import-modal .dropzone-hint { color: var(--muted); font-size: 0.82rem; line-height: 1.45; } + +/* Folder path row */ +#import-modal .field-row { display: flex; gap: 8px; align-items: stretch; } +#import-modal .field-row input { flex: 1; } +#import-modal #import-folder-path { + font-family: "SF Mono", "Menlo", monospace; font-size: 0.85rem; +} + +/* Manifest line */ +#import-modal .manifest { + margin: 14px 0 0; font-size: 0.88rem; color: var(--muted); + max-height: 30vh; overflow: auto; +} +#import-modal .manifest:empty { margin: 0; } +/* Breathing room above the pane's action button, regardless of whether + the manifest line is populated (it collapses when empty). */ +#import-modal .tab-pane > .btn { margin-top: 20px; } + +/* Buttons (scoped to the modal) */ +#import-modal .btn { + padding: 9px 16px; border-radius: 8px; + border: 1px solid var(--border); background: var(--panel-2); color: var(--text); + font-weight: 500; font-size: 0.92rem; cursor: pointer; + transition: background 140ms ease, border-color 140ms ease, filter 140ms ease; +} +#import-modal .btn:hover { background: var(--panel-3); border-color: var(--border-hover); } +#import-modal .btn.primary { + background: var(--accent); border-color: var(--accent); color: var(--on-accent); + font-weight: 600; +} +#import-modal .btn.primary:hover { filter: brightness(1.07); } +#import-modal .btn:disabled { opacity: 0.45; cursor: not-allowed; filter: none; } +#import-modal .btn.hidden { display: none; } + +/* Progress */ +#import-modal .progress { margin-top: 18px; } +#import-modal .progress.hidden { display: none; } +#import-modal .progress .bar { + height: 8px; background: var(--panel-2); border: 1px solid var(--border); + border-radius: 999px; overflow: hidden; +} +#import-modal .progress .bar span { + display: block; height: 100%; width: 0; border-radius: 999px; + background: linear-gradient(90deg, var(--accent-85), var(--accent)); + transition: width 220ms ease; +} +#import-modal #import-status { + margin: 10px 0 0; font-size: 0.85rem; color: var(--muted); + font-family: "SF Mono", "Menlo", monospace; +} + +/* Summary card */ +#import-modal .summary { + margin-top: 18px; padding: 14px 16px; + background: var(--panel-2); border: 1px solid var(--border); border-radius: 10px; + font-size: 0.9rem; line-height: 1.6; color: var(--text); +} +#import-modal .summary.hidden { display: none; } + +/* "Import recordings" launcher in the Download-manager header */ +.import-launch { + margin-left: auto; align-self: center; + display: inline-flex; align-items: center; gap: 7px; + padding: 7px 14px; border-radius: 8px; + background: var(--accent-08); border: 1px solid var(--accent-35); color: var(--accent); + font-weight: 600; font-size: 0.9rem; cursor: pointer; + transition: background 140ms ease, border-color 140ms ease; +} +.import-launch:hover { background: var(--accent-18); border-color: var(--accent); } +.import-launch svg { flex-shrink: 0; } From 1cf88588a0c49192105d07780a5371a6f9d515ad Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Fri, 5 Jun 2026 23:05:50 +0100 Subject: [PATCH 17/21] feat(logs): persistent application log with a Logs tab Replace the ephemeral in-DOM "Event log" on the downloads page with a durable, filterable log backed by a new app_log SQLite table. - DBLogHandler on the root logger captures INFO+ from viofosync* loggers (WARNING+ from everything else). emit() only enqueues; an async drain task batch-inserts off-loop, prunes to a 50k-row cap, and broadcasts each row over the existing WebSocket hub as {"type":"log"}. - GET /api/logs serves level/logger/search-filtered, before-cursor paginated history (auth-gated, fully parameterised queries). - New "Logs" tab live-tails new rows and loads history with expandable tracebacks, level filter (default Warning), and a soft row cap; the old event-log panel is removed. - Also folds in a small MQTT settings nav-label clarification. Co-Authored-By: Claude Opus 4.8 --- tests/test_log_store.py | 262 ++++++++++++++++++++++++++++++++++++++ tests/test_logs_api.py | 136 ++++++++++++++++++++ web/app.py | 37 +++++- web/db.py | 11 ++ web/routers/logs.py | 36 ++++++ web/services/log_store.py | 247 +++++++++++++++++++++++++++++++++++ web/static/app.js | 196 ++++++++++++++++++++++------ web/static/index.html | 32 ++++- web/static/styles.css | 71 +++++++---- 9 files changed, 957 insertions(+), 71 deletions(-) create mode 100644 tests/test_log_store.py create mode 100644 tests/test_logs_api.py create mode 100644 web/routers/logs.py create mode 100644 web/services/log_store.py diff --git a/tests/test_log_store.py b/tests/test_log_store.py new file mode 100644 index 0000000..e0bb806 --- /dev/null +++ b/tests/test_log_store.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +from web.db import Database + + +def test_app_log_table_created(tmp_path) -> None: + db = Database(str(tmp_path / "t.db")) + with db.conn() as c: + cols = { + row["name"] + for row in c.execute("PRAGMA table_info(app_log)").fetchall() + } + assert cols == { + "id", "ts", "levelno", "level", "logger", "message", "exc_text", + } + + +import asyncio +import logging +from contextlib import suppress + +import pytest + +from web.services import log_store +from web.services.log_store import DBLogHandler + + +def _record(name, level, msg, *, exc_info=None) -> logging.LogRecord: + return logging.LogRecord( + name=name, level=level, pathname=__file__, lineno=1, + msg=msg, args=None, exc_info=exc_info, + ) + + +async def _drain_once(handler: DBLogHandler) -> None: + """Let the bound drain task process the queued records.""" + for _ in range(50): + await asyncio.sleep(0.01) + if handler._queue is not None and handler._queue.empty(): + # one more tick so the in-flight batch finishes inserting + await asyncio.sleep(0.02) + return + + +@pytest.fixture +async def bound_handler(tmp_path): + from web.db import Database + db = Database(str(tmp_path / "t.db")) + sent: list = [] + + async def broadcast(ev): + sent.append(ev) + + h = DBLogHandler() + h.bind(db, broadcast, asyncio.get_running_loop()) + task = asyncio.create_task(h.run()) + try: + yield h, db, sent + finally: + task.cancel() + with suppress(asyncio.CancelledError): + await task + + +async def test_handler_persists_and_broadcasts(bound_handler) -> None: + h, db, sent = bound_handler + h.handle(_record("viofosync.test", logging.WARNING, "boom")) + await _drain_once(h) + with db.conn() as c: + rows = [ + tuple(r) for r in c.execute( + "SELECT level, logger, message FROM app_log" + ).fetchall() + ] + assert rows == [("WARNING", "viofosync.test", "boom")] + logs = [e for e in sent if e.get("type") == "log"] + assert logs and logs[0]["message"] == "boom" and logs[0]["id"] >= 1 + + +async def test_scope_filter(bound_handler) -> None: + h, db, _ = bound_handler + h.handle(_record("httpx", logging.INFO, "chatter")) # dropped + h.handle(_record("httpx", logging.WARNING, "uh oh")) # kept (>=WARNING) + h.handle(_record("viofosync.x", logging.INFO, "ours")) # kept (our ns) + await _drain_once(h) + with db.conn() as c: + msgs = { + r["message"] for r in c.execute( + "SELECT message FROM app_log" + ).fetchall() + } + assert msgs == {"uh oh", "ours"} + + +async def test_exc_text_captured(bound_handler) -> None: + h, db, _ = bound_handler + try: + raise ValueError("kaboom") + except ValueError: + import sys + h.handle(_record( + "viofosync.x", logging.ERROR, "failed", exc_info=sys.exc_info() + )) + await _drain_once(h) + with db.conn() as c: + exc = c.execute("SELECT exc_text FROM app_log").fetchone()["exc_text"] + assert exc is not None and "ValueError: kaboom" in exc + + +async def test_prune_keeps_newest(bound_handler, monkeypatch) -> None: + h, db, _ = bound_handler + monkeypatch.setattr(log_store, "APP_LOG_MAX_ROWS", 5) + with db.write() as c: + for i in range(12): + c.execute( + "INSERT INTO app_log (ts, levelno, level, logger, message) " + "VALUES (?, ?, ?, ?, ?)", + (float(i), 30, "WARNING", "viofosync.x", f"m{i}"), + ) + h._prune() + with db.conn() as c: + msgs = [ + r["message"] for r in c.execute( + "SELECT message FROM app_log ORDER BY id" + ).fetchall() + ] + assert msgs == [f"m{i}" for i in range(7, 12)] # newest 5 kept + + +async def test_reentrant_log_during_broadcast_does_not_hang(tmp_path) -> None: + """A log emitted *during* a broadcast (the recursion hazard) must be + captured too, without deadlocking the drain task.""" + from web.db import Database + db = Database(str(tmp_path / "t.db")) + fired = {"once": False} + + async def broadcast(ev): + if not fired["once"]: + fired["once"] = True + logging.getLogger("viofosync.during").warning("nested") + + h = DBLogHandler() + logging.getLogger().addHandler(h) + h.bind(db, broadcast, asyncio.get_running_loop()) + task = asyncio.create_task(h.run()) + try: + logging.getLogger("viofosync.first").warning("outer") + for _ in range(100): + await asyncio.sleep(0.01) + with db.conn() as c: + n = c.execute("SELECT COUNT(*) AS n FROM app_log").fetchone()["n"] + if n >= 2: + break + with db.conn() as c: + msgs = { + r["message"] for r in c.execute( + "SELECT message FROM app_log" + ).fetchall() + } + assert {"outer", "nested"} <= msgs + finally: + logging.getLogger().removeHandler(h) + task.cancel() + with suppress(asyncio.CancelledError): + await task + + +async def test_overbroad_logger_name_not_captured(bound_handler) -> None: + h, db, _ = bound_handler + h.handle(_record("viofosyncx", logging.INFO, "no")) # not our namespace + h.handle(_record("viofosync.y", logging.INFO, "yes")) # our namespace + await _drain_once(h) + with db.conn() as c: + msgs = { + r["message"] for r in c.execute( + "SELECT message FROM app_log" + ).fetchall() + } + assert msgs == {"yes"} + + +async def test_enqueue_drops_when_queue_full(tmp_path, monkeypatch) -> None: + from web.db import Database + monkeypatch.setattr(log_store, "_QUEUE_MAXSIZE", 3) + db = Database(str(tmp_path / "t.db")) + + async def broadcast(ev): + pass + + h = DBLogHandler() + h.bind(db, broadcast, asyncio.get_running_loop()) # bound, but no run() task draining + for i in range(5): + h._enqueue({ + "ts": 0.0, "levelno": 30, "level": "WARNING", + "logger": "viofosync.x", "message": f"m{i}", "exc_text": None, + }) + assert h._queue.qsize() == 3 + assert h._dropped == 2 + + +def _seed(db) -> None: + rows = [ + (1.0, 20, "INFO", "viofosync.scanner", "scan start"), + (2.0, 30, "WARNING", "viofosync.sync_worker", "retry 1"), + (3.0, 40, "ERROR", "viofosync.sync_worker", "download failed"), + (4.0, 20, "INFO", "viofosync.geocode", "cache hit"), + ] + with db.write() as c: + for r in rows: + c.execute( + "INSERT INTO app_log (ts, levelno, level, logger, message) " + "VALUES (?, ?, ?, ?, ?)", + r, + ) + + +def test_query_defaults_to_warning_plus_newest_first(tmp_path) -> None: + from web.db import Database + db = Database(str(tmp_path / "t.db")) + _seed(db) + out = log_store.query_logs(db) # min_levelno defaults to WARNING + assert [e["message"] for e in out] == ["download failed", "retry 1"] + + +def test_query_info_includes_everything(tmp_path) -> None: + from web.db import Database + db = Database(str(tmp_path / "t.db")) + _seed(db) + out = log_store.query_logs(db, min_levelno=20) + assert len(out) == 4 + + +def test_query_filters_logger_and_message(tmp_path) -> None: + from web.db import Database + db = Database(str(tmp_path / "t.db")) + _seed(db) + by_logger = log_store.query_logs(db, min_levelno=20, logger="geocode") + assert [e["message"] for e in by_logger] == ["cache hit"] + by_msg = log_store.query_logs(db, min_levelno=20, q="failed") + assert [e["message"] for e in by_msg] == ["download failed"] + + +def test_query_before_paginates(tmp_path) -> None: + from web.db import Database + db = Database(str(tmp_path / "t.db")) + _seed(db) + page1 = log_store.query_logs(db, min_levelno=20, limit=2) + assert [e["id"] for e in page1] == [4, 3] + page2 = log_store.query_logs( + db, min_levelno=20, limit=2, before=page1[-1]["id"] + ) + assert [e["id"] for e in page2] == [2, 1] + + +def test_query_clamps_limit(tmp_path) -> None: + from web.db import Database + db = Database(str(tmp_path / "t.db")) + _seed(db) + # limit below 1 clamps up to 1 + assert len(log_store.query_logs(db, min_levelno=20, limit=0)) == 1 + # absurd limit clamps down to 1000 (still returns all 4 seeded rows) + assert len(log_store.query_logs(db, min_levelno=20, limit=10_000)) == 4 diff --git a/tests/test_logs_api.py b/tests/test_logs_api.py new file mode 100644 index 0000000..6f786c6 --- /dev/null +++ b/tests/test_logs_api.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import logging +import time +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture +def authed_client(tmp_config_dir: Path, tmp_recordings_dir: Path, monkeypatch): + from web import app as app_mod + from web import settings as settings_mod + monkeypatch.setenv("VIOFOSYNC_RESTART_DISABLED", "1") + settings_mod.reset_for_tests() + application = app_mod.create_app() + with TestClient(application) as c: + c.post("/setup", data={ + "address": "192.168.1.230", + "password": "twelve-chars-min!", + "confirm": "twelve-chars-min!", + }) + csrf = c.get("/api/auth/csrf").json()["csrf"] + c.headers.update({"x-csrf-token": csrf}) + # Detach the live log handler so startup/runtime records can't + # land in app_log after we seed, then wait for the drain to flush + # whatever it already enqueued. We clear the table and reset the + # AUTOINCREMENT sequence so the seeded rows are deterministically + # ids 1..3 — the id/count assertions below depend on that. + handler = getattr(c.app.state, "log_handler", None) + if handler is not None: + logging.getLogger().removeHandler(handler) + + def _row_count() -> int: + with c.app.state.db.conn() as conn: + return conn.execute( + "SELECT COUNT(*) FROM app_log" + ).fetchone()[0] + + prev, stable = -1, 0 + for _ in range(100): # up to ~2s for the async drain to settle + n = _row_count() + stable = stable + 1 if n == prev else 0 + if stable >= 3: + break + prev = n + time.sleep(0.02) + with c.app.state.db.write() as conn: + conn.execute("DELETE FROM app_log") + conn.execute( + "DELETE FROM sqlite_sequence WHERE name = 'app_log'" + ) + # Seed rows directly so the test does not depend on drain timing. + with c.app.state.db.write() as conn: + for r in [ + (1.0, 20, "INFO", "viofosync.scanner", "scan start"), + (2.0, 30, "WARNING", "viofosync.sync_worker", "retry 1"), + (3.0, 40, "ERROR", "viofosync.sync_worker", "boom"), + ]: + conn.execute( + "INSERT INTO app_log " + "(ts, levelno, level, logger, message) " + "VALUES (?, ?, ?, ?, ?)", + r, + ) + yield c + + +def test_logs_requires_auth(tmp_config_dir, tmp_recordings_dir, monkeypatch): + from web import app as app_mod + from web import settings as settings_mod + monkeypatch.setenv("VIOFOSYNC_RESTART_DISABLED", "1") + settings_mod.reset_for_tests() + with TestClient(app_mod.create_app()) as c: + c.post("/setup", data={ + "address": "192.168.1.230", + "password": "twelve-chars-min!", + "confirm": "twelve-chars-min!", + }) + c.cookies.clear() + r = c.get("/api/logs") + assert r.status_code == 401 + + +def test_logs_default_warning_plus(authed_client): + r = authed_client.get("/api/logs") + assert r.status_code == 200 + msgs = [e["message"] for e in r.json()["entries"]] + assert msgs == ["boom", "retry 1"] + + +def test_logs_level_info_includes_all(authed_client): + r = authed_client.get("/api/logs?level=INFO") + assert len(r.json()["entries"]) == 3 + + +def test_logs_filter_logger_and_q(authed_client): + r = authed_client.get("/api/logs?level=INFO&logger=scanner") + assert [e["message"] for e in r.json()["entries"]] == ["scan start"] + r = authed_client.get("/api/logs?level=INFO&q=retry") + assert [e["message"] for e in r.json()["entries"]] == ["retry 1"] + + +def test_logs_before_pagination(authed_client): + page1 = authed_client.get("/api/logs?level=INFO&limit=2").json()["entries"] + assert [e["id"] for e in page1] == [3, 2] + before = page1[-1]["id"] + page2 = authed_client.get( + f"/api/logs?level=INFO&limit=2&before={before}" + ).json()["entries"] + assert [e["id"] for e in page2] == [1] + + +def test_emitted_log_reaches_api(tmp_config_dir, tmp_recordings_dir, monkeypatch): + """A real logging call after startup is persisted and served.""" + from web import app as app_mod + from web import settings as settings_mod + monkeypatch.setenv("VIOFOSYNC_RESTART_DISABLED", "1") + settings_mod.reset_for_tests() + with TestClient(app_mod.create_app()) as c: + c.post("/setup", data={ + "address": "192.168.1.230", + "password": "twelve-chars-min!", + "confirm": "twelve-chars-min!", + }) + logging.getLogger("viofosync.endtoend").warning("hello-from-test") + found = [] + for _ in range(100): # up to ~2s for the async drain + time.sleep(0.02) + r = c.get("/api/logs?level=INFO&q=hello-from-test") + if r.status_code == 200 and r.json()["entries"]: + found = r.json()["entries"] + break + assert found and found[0]["message"] == "hello-from-test" + assert found[0]["logger"] == "viofosync.endtoend" diff --git a/web/app.py b/web/app.py index 121ff7c..b0c933e 100644 --- a/web/app.py +++ b/web/app.py @@ -14,7 +14,7 @@ import logging import os -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from fastapi import FastAPI from fastapi.responses import FileResponse, Response @@ -33,6 +33,7 @@ from .routers import setup as setup_router from .routers import storage as storage_router from .routers import imports as imports_router +from .routers import logs as logs_router from .services import retention as _ret_mod from .services import scanner from .services.exporter import ( @@ -43,6 +44,7 @@ from .services.geocode import GeocodeService from .services.download_session import DownloadSession from .services.hub import Hub +from .services.log_store import DBLogHandler from .services.mqtt import MqttService from .services.sync_worker import SyncWorker from .setup_mode import SetupModeMiddleware @@ -151,6 +153,17 @@ async def _background_retention() -> None: settings_provider=provider, session=app.state.download_session, ) + # Now that db + hub + loop exist, let the log handler persist and + # live-broadcast records. Records logged earlier in startup were + # buffered by the handler and flush when the drain task starts. + log_handler = getattr(app.state, "log_handler", None) + if log_handler is not None: + log_handler.bind( + app.state.db, + app.state.hub.broadcast, + asyncio.get_running_loop(), + ) + app.state.log_drain_task = asyncio.create_task(log_handler.run()) # Compute initial sync_status so the very first WebSocket snapshot # and the first MQTT publish carry the right value without waiting # for the sync worker's first cycle. @@ -226,6 +239,18 @@ def _on_mqtt_settings_change(keys, snap): task.cancel() await app.state.sync_worker.stop() await app.state.export_worker.stop() + drain = getattr(app.state, "log_drain_task", None) + if drain is not None and not drain.done(): + drain.cancel() + # Await the cancellation so the task unwinds before the loop + # tears down — bare cancel() only stays warning-clean while + # run() happens to be parked in queue.get(); awaiting it makes + # that independent of where cancellation lands. + with suppress(asyncio.CancelledError): + await drain + log_handler = getattr(app.state, "log_handler", None) + if log_handler is not None: + logging.getLogger().removeHandler(log_handler) def create_app() -> FastAPI: @@ -246,6 +271,15 @@ def create_app() -> FastAPI: redoc_url=None, ) + # Persist INFO+ from our loggers (and WARNING+ from everything) into + # the app_log table for the Logs tab. The handler only enqueues here; + # lifespan binds the DB/hub/loop and starts the drain task. basicConfig + # above ran with force=True, so any handler from a previous create_app + # (in tests) was already removed — no accumulation. + log_handler = DBLogHandler() + logging.getLogger().addHandler(log_handler) + app.state.log_handler = log_handler + app.add_middleware(SetupModeMiddleware) app.include_router(auth_router.router) @@ -258,6 +292,7 @@ def create_app() -> FastAPI: app.include_router(mqtt_router.router) app.include_router(storage_router.router) app.include_router(imports_router.router) + app.include_router(logs_router.router) # Static SPA — served at / with an explicit index.html fall-through # so the SPA's hash-router owns everything that isn't /api/*. diff --git a/web/db.py b/web/db.py index c2048c7..0845b76 100644 --- a/web/db.py +++ b/web/db.py @@ -161,6 +161,17 @@ def migrate_legacy_db_path(new_path: str) -> None: fetched_at INTEGER NOT NULL, PRIMARY KEY (lat_key, lon_key) ); + +CREATE TABLE IF NOT EXISTS app_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts REAL NOT NULL, -- record.created (unix seconds, fractional) + levelno INTEGER NOT NULL, -- 10/20/30/40/50; "WARNING+" = levelno >= 30 + level TEXT NOT NULL, -- 'INFO','WARNING','ERROR',... + logger TEXT NOT NULL, -- record.name, e.g. 'viofosync.sync_worker' + message TEXT NOT NULL, -- record.getMessage() + exc_text TEXT -- formatted traceback, NULL when none +); +CREATE INDEX IF NOT EXISTS idx_app_log_levelno ON app_log(levelno, id DESC); """ diff --git a/web/routers/logs.py b/web/routers/logs.py new file mode 100644 index 0000000..4d25f12 --- /dev/null +++ b/web/routers/logs.py @@ -0,0 +1,36 @@ +"""Persistent application log endpoint.""" +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query, Request + +from ..auth import require_session +from ..services import log_store + +router = APIRouter( + prefix="/api", + tags=["logs"], + dependencies=[Depends(require_session)], +) + + +@router.get("/logs") +def list_logs( + request: Request, + level: str = Query("WARNING"), + logger: str | None = Query(None), + q: str | None = Query(None), + before: int | None = Query(None), + limit: int = Query(200, ge=1, le=1000), +) -> dict: + # Unknown level strings fall back to WARNING (the default view). + # Reference the table rather than a bare literal so the two can't drift. + min_levelno = log_store.LEVELS.get(level.upper(), log_store.LEVELS["WARNING"]) + entries = log_store.query_logs( + request.app.state.db, + min_levelno=min_levelno, + logger=logger.strip() if logger else None, + q=q.strip() if q else None, + before=before, + limit=limit, + ) + return {"entries": entries} diff --git a/web/services/log_store.py b/web/services/log_store.py new file mode 100644 index 0000000..6d82bc3 --- /dev/null +++ b/web/services/log_store.py @@ -0,0 +1,247 @@ +"""Persistent application log. + +Captures records from the standard ``logging`` framework into the +``app_log`` table and live-broadcasts each one over the WebSocket hub, +so the UI's Logs tab shows a durable, filterable history instead of the +old ephemeral in-DOM event log. + +Capture is decoupled from I/O: ``DBLogHandler.emit`` only enqueues onto +the event loop; a single async ``run`` task batch-inserts and broadcasts. +This keeps ``emit`` non-blocking on whatever thread logged (sync worker, +export worker, uvicorn) and makes re-entrancy impossible — a DB error in +the drain task that itself logs just enqueues another record rather than +recursing into a synchronous write. +""" +from __future__ import annotations + +import asyncio +import collections +import logging +import sys +from typing import Any, Awaitable, Callable, Deque, Dict, List, Optional + +# Keep newest this-many rows; older rows are pruned. A module-level +# constant (not a setting yet) so tests can monkeypatch it. +APP_LOG_MAX_ROWS = 50_000 + +# INFO+ is persisted from our own loggers; everything else only at +# WARNING+ (keeps third-party INFO chatter — httpx, uvicorn — out). +_APP_NAMESPACES = ("viofosync", "viofosync_lib") +_APP_PREFIXES = tuple(ns + "." for ns in _APP_NAMESPACES) + +# Rows inserted between prune sweeps. +_PRUNE_EVERY = 200 + +# Cap the live queue and per-transaction batch so a burst of logging +# can't grow memory or transaction size without bound. Overflow drops +# the newest record (and notes the drop on stderr) rather than blocking +# the thread that logged. +_QUEUE_MAXSIZE = 10_000 +_MAX_BATCH = 1_000 + +# Name -> numeric level, for the API's `level` filter. +LEVELS = { + "DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50, +} + +BroadcastFn = Callable[[Dict[str, Any]], Awaitable[None]] + + +def _should_capture(record: logging.LogRecord) -> bool: + if record.levelno >= logging.WARNING: + return True + name = record.name + return name in _APP_NAMESPACES or name.startswith(_APP_PREFIXES) + + +class _CaptureFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: # noqa: A003 + return _should_capture(record) + + +class DBLogHandler(logging.Handler): + """Root-logger handler that persists records to ``app_log``. + + ``emit`` is cheap and thread-safe: it formats the record and hands + it to the event loop. Nothing touches the DB until the async + ``run`` task drains the queue. + """ + + def __init__(self) -> None: + super().__init__(level=logging.INFO) + self.addFilter(_CaptureFilter()) + # Records seen before the loop is bound buffer here and flush + # when ``run`` starts. Startup contract: callers must invoke + # ``bind()`` and start ``run()`` back-to-back on the loop thread; + # a record logged from another thread in that tiny pre-bind window + # buffers here, and the bounded startup flush may drop a record + # logged in the microsecond between the flush and the clear. + self._pending: Deque[Dict[str, Any]] = collections.deque(maxlen=1000) + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._queue: Optional[asyncio.Queue] = None + self._db: Any = None + self._broadcast: Optional[BroadcastFn] = None + self._dropped = 0 + + # -- logging.Handler API -- + + def emit(self, record: logging.LogRecord) -> None: + try: + payload = self._to_payload(record) + except Exception: # never let logging raise into the caller + return + loop, queue = self._loop, self._queue + if loop is None or queue is None: + self._pending.append(payload) + return + try: + loop.call_soon_threadsafe(self._enqueue, payload) + except RuntimeError: + # Loop is closed (shutdown) — drop the record. + pass + + def _enqueue(self, payload: Dict[str, Any]) -> None: + q = self._queue + if q is None: # pragma: no cover — bound before emit schedules this + return + try: + q.put_nowait(payload) + except asyncio.QueueFull: + self._dropped += 1 + # Surface the overflow without re-entering the log path. + if self._dropped % 1000 == 1: + print( + f"app_log queue full; dropped {self._dropped} record(s)", + file=sys.stderr, + ) + + @staticmethod + def _to_payload(record: logging.LogRecord) -> Dict[str, Any]: + exc_text = None + if record.exc_info: + exc_text = logging.Formatter().formatException(record.exc_info) + return { + "ts": record.created, + "levelno": record.levelno, + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "exc_text": exc_text, + } + + # -- wiring -- + + def bind( + self, + db: Any, + broadcast: BroadcastFn, + loop: asyncio.AbstractEventLoop, + ) -> None: + """Attach the DB, hub broadcast coroutine, and loop. Must be + called from inside the running loop (creates the asyncio.Queue).""" + self._db = db + self._broadcast = broadcast + self._loop = loop + self._queue = asyncio.Queue(maxsize=_QUEUE_MAXSIZE) + + async def run(self) -> None: + """Drain loop. Cancel the task to stop.""" + assert self._queue is not None, "bind() before run()" + if self._pending: + await self._drain_batch(list(self._pending)) + self._pending.clear() + await asyncio.to_thread(self._prune) + since_prune = 0 + while True: + payload = await self._queue.get() + batch = [payload] + while len(batch) < _MAX_BATCH: + try: + batch.append(self._queue.get_nowait()) + except asyncio.QueueEmpty: + break + await self._drain_batch(batch) + since_prune += len(batch) + if since_prune >= _PRUNE_EVERY: + since_prune = 0 + await asyncio.to_thread(self._prune) + + # -- internals -- + + async def _drain_batch(self, batch: List[Dict[str, Any]]) -> None: + rows = await asyncio.to_thread(self._insert_batch, batch) + if self._broadcast is None: + return + for log_id, p in rows: + await self._broadcast({"type": "log", "id": log_id, **p}) + + def _insert_batch( + self, batch: List[Dict[str, Any]] + ) -> List[tuple]: + rows: List[tuple] = [] + try: + with self._db.write() as c: + for p in batch: + cur = c.execute( + "INSERT INTO app_log " + "(ts, levelno, level, logger, message, exc_text) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + p["ts"], p["levelno"], p["level"], + p["logger"], p["message"], p["exc_text"], + ), + ) + rows.append((cur.lastrowid, p)) + except Exception as e: # pragma: no cover — report off the log path + print(f"app_log insert failed: {e}", file=sys.stderr) + return [] + return rows + + def _prune(self) -> None: + try: + with self._db.write() as c: + c.execute( + "DELETE FROM app_log WHERE id <= " + "(SELECT MAX(id) FROM app_log) - ?", + (APP_LOG_MAX_ROWS,), + ) + except Exception as e: # pragma: no cover + print(f"app_log prune failed: {e}", file=sys.stderr) + + +def query_logs( + db: Any, + *, + min_levelno: int = logging.WARNING, + logger: Optional[str] = None, + q: Optional[str] = None, + before: Optional[int] = None, + limit: int = 200, +) -> List[Dict[str, Any]]: + """Return persisted log rows, newest-first. + + ``before`` (an id) and ``limit`` drive "load older" pagination. + ``logger`` / ``q`` are case-insensitive substring matches. + """ + limit = max(1, min(int(limit), 1000)) + clauses = ["levelno >= ?"] + params: List[Any] = [int(min_levelno)] + if logger: + clauses.append("logger LIKE ?") + params.append(f"%{logger}%") + if q: + clauses.append("message LIKE ?") + params.append(f"%{q}%") + if before is not None: + clauses.append("id < ?") + params.append(int(before)) + where = " AND ".join(clauses) + params.append(limit) + # NB: levelno is a range filter, so SQLite serves this by scanning id + # DESC (the PK) rather than idx_app_log_levelno; fine within the row cap. + sql = ( + "SELECT id, ts, levelno, level, logger, message, exc_text " + f"FROM app_log WHERE {where} ORDER BY id DESC LIMIT ?" + ) + with db.conn() as c: + return [dict(r) for r in c.execute(sql, params).fetchall()] diff --git a/web/static/app.js b/web/static/app.js index d46509d..1955782 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -31,6 +31,8 @@ const state = { // helpers (fmtDistance) don't need to read from settingsState // (which is only loaded when the Settings tab is visited). distanceUnits: "km", + logsFilter: null, // { level, logger, q } currently shown in Logs tab + logsOldestId: null, // smallest id loaded, for "Load older" pagination }; // ---------- CSS variable bridge ---------- @@ -237,6 +239,8 @@ function routeTo(hash) { }); document.getElementById("view-archive").hidden = tab !== "archive"; document.getElementById("view-downloads").hidden = tab !== "downloads"; + const logsView = document.getElementById("view-logs"); + if (logsView) logsView.hidden = tab !== "logs"; const settingsView = document.getElementById("view-settings"); if (settingsView) settingsView.hidden = tab !== "settings"; if (tab === "archive") { @@ -247,6 +251,7 @@ function routeTo(hash) { stopArchiveAutoRefresh(); } if (tab === "downloads") loadQueue(); + if (tab === "logs") loadLogs(); if (tab === "settings") loadSettings(); } @@ -1981,6 +1986,152 @@ function refreshQueueIfVisible() { } } +// ---------- Logs tab ---------- + +const LOG_LEVELNO = { + DEBUG: 10, INFO: 20, WARNING: 30, ERROR: 40, CRITICAL: 50, +}; +const LOGS_PAGE = 200; // server page size for history fetches +const LOGS_MAX_ROWS = 1000; // cap live-tail DOM growth over a long session + +function logsFilterParams() { + return { + level: document.getElementById("logs-level").value, + logger: document.getElementById("logs-logger").value.trim(), + q: document.getElementById("logs-q").value.trim(), + }; +} + +function logsQueryString(f, extra = {}) { + const params = new URLSearchParams({ level: f.level, limit: String(LOGS_PAGE), ...extra }); + if (f.logger) params.set("logger", f.logger); + if (f.q) params.set("q", f.q); + return params.toString(); +} + +function renderLogRow(e) { + const row = document.createElement("div"); + row.className = "log-line log-" + (e.level || "").toLowerCase(); + row.dataset.id = e.id; + + const head = document.createElement("div"); + head.className = "log-head"; + + const ts = document.createElement("span"); + ts.className = "log-ts"; + ts.textContent = new Date(e.ts * 1000).toLocaleTimeString(); + + const lvl = document.createElement("span"); + lvl.className = "log-level"; + lvl.textContent = e.level; + + const logger = document.createElement("span"); + logger.className = "log-logger"; + logger.textContent = e.logger; + + const msg = document.createElement("span"); + msg.className = "log-msg"; + msg.textContent = e.message; + + head.append(ts, lvl, logger, msg); + row.appendChild(head); + + if (e.exc_text) { + head.classList.add("has-exc"); + head.addEventListener("click", () => row.classList.toggle("open")); + const pre = document.createElement("pre"); + pre.className = "log-exc"; + pre.textContent = e.exc_text; + row.appendChild(pre); + } + return row; +} + +async function loadLogs() { + const f = logsFilterParams(); + state.logsFilter = f; + const list = document.getElementById("logs-list"); + const older = document.getElementById("logs-older"); + list.innerHTML = ""; + let entries = []; + try { + entries = (await api(`/api/logs?${logsQueryString(f)}`)).entries; + } catch { return; } + if (!entries.length) { + const empty = document.createElement("div"); + empty.className = "logs-empty"; + empty.textContent = "No log entries match the current filter."; + list.appendChild(empty); + } else { + for (const e of entries) list.appendChild(renderLogRow(e)); + } + state.logsOldestId = entries.length ? entries[entries.length - 1].id : null; + // A short page means there is no older history behind it. + if (older) { + const more = entries.length >= LOGS_PAGE; + older.disabled = !more; + older.textContent = more ? "Load older" : "No older entries"; + } +} + +async function loadOlderLogs() { + if (!state.logsOldestId) return; + const f = state.logsFilter || logsFilterParams(); + const qs = logsQueryString(f, { before: String(state.logsOldestId) }); + const older = document.getElementById("logs-older"); + let entries = []; + try { entries = (await api(`/api/logs?${qs}`)).entries; } + catch { return; } + const list = document.getElementById("logs-list"); + for (const e of entries) list.appendChild(renderLogRow(e)); + if (entries.length) state.logsOldestId = entries[entries.length - 1].id; + if (older && entries.length < LOGS_PAGE) { + older.disabled = true; + older.textContent = "No older entries"; + } +} + +function logMatchesFilter(e, f) { + const min = LOG_LEVELNO[f.level] || 30; + const lvl = e.levelno || LOG_LEVELNO[e.level] || 0; + if (lvl < min) return false; + if (f.logger && !(e.logger || "").includes(f.logger)) return false; + if (f.q && !(e.message || "").toLowerCase().includes(f.q.toLowerCase())) { + return false; + } + return true; +} + +function logsLive(e) { + const view = document.getElementById("view-logs"); + const list = document.getElementById("logs-list"); + if (!list || !view || view.hidden) return; + // NB: state.logsFilter only resyncs when loadLogs() runs (on a toolbar + // `change` or Refresh), so for a few seconds after editing a search box + // (which commits on blur/Enter) live rows are matched against the prior + // filter; loadLogs() re-renders cleanly on commit. + const f = state.logsFilter || logsFilterParams(); + if (!logMatchesFilter(e, f)) return; + const placeholder = list.querySelector(".logs-empty"); + if (placeholder) placeholder.remove(); + const atTop = list.scrollTop <= 4; + list.insertBefore(renderLogRow(e), list.firstChild); + if (atTop) list.scrollTop = 0; + // Bound live-tail growth (the old event-log panel capped at 200). Trim the + // oldest rows and keep logsOldestId at the oldest visible id so paging stays + // contiguous. + if (list.children.length > LOGS_MAX_ROWS) { + while (list.children.length > LOGS_MAX_ROWS) list.removeChild(list.lastChild); + if (list.lastChild) state.logsOldestId = Number(list.lastChild.dataset.id); + } +} + +document.getElementById("logs-refresh").addEventListener("click", loadLogs); +document.getElementById("logs-older").addEventListener("click", loadOlderLogs); +for (const id of ["logs-level", "logs-logger", "logs-q"]) { + document.getElementById(id).addEventListener("change", loadLogs); +} + function openSocket() { if (state.ws) { try { state.ws.close(); } catch {} } const proto = location.protocol === "https:" ? "wss:" : "ws:"; @@ -1993,49 +2144,7 @@ function openSocket() { }); } -const MAX_LOG_ENTRIES = 200; - -function appendLog(ev) { - const container = document.getElementById("log-entries"); - if (!container) return; - // Per-chunk progress is already shown live in the progress bar - // above (downloads) or the Export jobs table (exports); mirroring - // it in the log buries everything else. - if (ev.type === "item_progress" || ev.type === "export_progress" - || ev.type === "session_stats") return; - const line = document.createElement("div"); - line.className = "log-line"; - const ts = new Date().toLocaleTimeString(); - let detail = ev.type; - if (ev.type === "item_started") detail = `item_started: ${ev.filename}`; - else if (ev.type === "item_finished") detail = `item_finished: ${ev.filename} ${ev.ok ? "ok" : ev.error || "failed"}`; - else if (ev.type === "sync_state") detail = `sync_state: running=${ev.running} paused=${ev.paused}`; - else if (ev.type === "queue_reconciled") detail = `queue_reconciled: +${ev.added || 0} added, ${ev.marked_gone || 0} gone`; - else if (ev.type === "dashcam_delete") { - if (ev.ok) detail = `dashcam_delete: ${ev.filename} ok`; - else if (ev.reason === "size_mismatch" - && ev.local_size != null && ev.remote_size != null) { - const delta = ev.local_size - ev.remote_size; - const sign = delta >= 0 ? "+" : ""; - detail = `dashcam_delete: ${ev.filename} skipped ` - + `(size_mismatch: local=${ev.local_size} dashcam=${ev.remote_size}, ${sign}${delta})`; - } - else detail = `dashcam_delete: ${ev.filename} skipped (${ev.reason || "unknown"})`; - } - else if (ev.type === "retention_deleted") { - detail = `retention_deleted: ${ev.filename} (${ev.reason})`; - } - else if (ev.type === "snapshot") detail = "snapshot (initial state)"; - line.innerHTML = `${ts} ${detail}`; - container.appendChild(line); - while (container.children.length > MAX_LOG_ENTRIES) { - container.removeChild(container.firstChild); - } - container.scrollTop = container.scrollHeight; -} - function handleEvent(ev) { - appendLog(ev); const statusEl = document.getElementById("sync-status"); const STATUS_LABEL = { downloading: "Downloading", @@ -2143,6 +2252,9 @@ function handleEvent(ev) { case "import_done": if (window.__importOnEvent) window.__importOnEvent(ev); break; + case "log": + logsLive(ev); + break; } } diff --git a/web/static/index.html b/web/static/index.html index d2c3e72..2382fae 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -33,6 +33,7 @@

Viofosync

@@ -170,11 +171,6 @@

Download manager

- -
- Event log -
-
+ +