From 5dffb198d0e2ba58132f57176b8a6c7332a49625 Mon Sep 17 00:00:00 2001 From: Brian Hynds Date: Thu, 25 Jun 2026 14:07:46 -0400 Subject: [PATCH 1/3] Support compact single-channel Viofo filenames --- tests/test_queue_day_grouping.py | 48 +++++++++++++++++++ tests/test_queue_filename_parsing.py | 18 +++++++ tests/test_scanner_compact_filename.py | 27 +++++++++++ viofosync_lib/_archive.py | 17 +++---- web/services/queue.py | 65 +++++++++++++++----------- web/services/scanner.py | 6 +-- 6 files changed, 141 insertions(+), 40 deletions(-) create mode 100644 tests/test_queue_day_grouping.py create mode 100644 tests/test_scanner_compact_filename.py diff --git a/tests/test_queue_day_grouping.py b/tests/test_queue_day_grouping.py new file mode 100644 index 0000000..2a228e5 --- /dev/null +++ b/tests/test_queue_day_grouping.py @@ -0,0 +1,48 @@ +"""Queue day grouping for compact Viofo filenames.""" +from __future__ import annotations + +import datetime as dt +import time + +from web.db import Database +from web.services import queue + + +def _recorded_at() -> int: + return int( + dt.datetime( + 2026, 6, 25, 17, 12, 42, tzinfo=dt.timezone.utc + ).timestamp() + ) + + +def test_compact_filename_uses_recorded_at_for_day_grouping(tmp_path): + db = Database(str(tmp_path / "test.db")) + now = int(time.time()) + with db.write() as c: + c.execute( + """ + INSERT INTO download_queue ( + filename, source_dir, remote_size, recorded_at, + camera, event_type, state, enqueued_at + ) VALUES (?,?,?,?,?,?,?,?) + """, + ( + "20260625171242_000086.MP4", + "/DCIM/Movie", + 1024, + _recorded_at(), + "F", + "normal", + "pending", + now, + ), + ) + + days = queue.list_days(db) + assert [d["day"] for d in days] == ["2026-06-25"] + + items = queue.list_day_items(db, "2026-06-25") + assert len(items) == 1 + assert items[0]["kind_camera"] == "F" + assert items[0]["kind_event"] == "normal" diff --git a/tests/test_queue_filename_parsing.py b/tests/test_queue_filename_parsing.py index cf92cfc..4646808 100644 --- a/tests/test_queue_filename_parsing.py +++ b/tests/test_queue_filename_parsing.py @@ -7,12 +7,30 @@ """ from __future__ import annotations +from viofosync_lib import downloaded_filename_re from web.services.queue import ( _camera_from_filename, _event_from_filename, ) +def test_compact_single_channel_filename_defaults_to_front_normal(): + filename = "20260625171242_000086.MP4" + assert _camera_from_filename(filename) == "F" + assert _event_from_filename(filename) == "normal" + + m = downloaded_filename_re.match(filename) + assert m is not None + assert m.group("year") == "2026" + assert m.group("month") == "06" + assert m.group("day") == "25" + assert m.group("hour") == "17" + assert m.group("minute") == "12" + assert m.group("second") == "42" + assert m.group("sequence") == "000086" + assert m.group("camera") is None + + def test_camera_front_rear(): assert _camera_from_filename("2026_0513_172152_000120F.MP4") == "F" assert _camera_from_filename("2026_0513_172152_000121R.MP4") == "R" diff --git a/tests/test_scanner_compact_filename.py b/tests/test_scanner_compact_filename.py new file mode 100644 index 0000000..8d40e7c --- /dev/null +++ b/tests/test_scanner_compact_filename.py @@ -0,0 +1,27 @@ +"""Scanner support for compact Viofo filenames.""" +from __future__ import annotations + +from web.db import Database +from web.services import scanner + + +def test_scanner_indexes_compact_single_channel_filename( + tmp_path, + tmp_recordings_dir, +): + db = Database(str(tmp_path / "test.db")) + day_dir = tmp_recordings_dir / "2026-06-25" + day_dir.mkdir() + filename = "20260625171242_000086.MP4" + (day_dir / filename).write_bytes(b"clip") + + count = scanner.scan(db, str(tmp_recordings_dir), "daily") + + assert count == 1 + with db.conn() as c: + row = c.execute("SELECT * FROM clip_index").fetchone() + assert row["basename"] == filename + assert row["group_name"] == "2026-06-25" + assert row["camera"] == "F" + assert row["event_type"] == "normal" + assert row["sequence"] == 86 diff --git a/viofosync_lib/_archive.py b/viofosync_lib/_archive.py index 3c754a6..6fcbea6 100644 --- a/viofosync_lib/_archive.py +++ b/viofosync_lib/_archive.py @@ -13,8 +13,6 @@ import re from collections import namedtuple -from .cameras import CAMERA_LETTERS - logger = logging.getLogger("viofosync_lib.archive") # Recording namedtuple matching Viofo's file information. @@ -32,19 +30,18 @@ "yearly": "[0-9][0-9][0-9][0-9]", } -# Downloaded recording filename glob pattern. The trailing -# letter is the camera (see cameras.py for the registry). +# Downloaded recording filename glob pattern. Some Viofo models write +# files as ``YYYYMMDDHHMMSS_NNNNNN.MP4`` without a camera suffix, while +# newer/multi-channel models use ``YYYY_MMDD_HHMMSS_NNNN[FRTI].MP4``. downloaded_filename_glob = ( - "[0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]" - "_[0-9][0-9][0-9][0-9][0-9][0-9]" - f"_*[{CAMERA_LETTERS}].MP4" + "[0-9][0-9][0-9][0-9]*_[0-9]*.MP4" ) # Downloaded recording filename regular expression. downloaded_filename_re = re.compile( - r"^(?P\d{4})_(?P\d{2})(?P\d{2})" - r"_(?P\d{2})(?P\d{2})(?P\d{2})" - r"_(?P\d+)(?P.+)\.MP4$", + r"^(?P\d{4})_?(?P\d{2})(?P\d{2})" + r"_?(?P\d{2})(?P\d{2})(?P\d{2})" + r"_(?P\d+)(?P[PE]?[FRTI])?\.MP4$", re.IGNORECASE, ) diff --git a/web/services/queue.py b/web/services/queue.py index d857cf2..87e4502 100644 --- a/web/services/queue.py +++ b/web/services/queue.py @@ -24,6 +24,7 @@ from dataclasses import dataclass from typing import Iterable, List, Optional +from viofosync_lib import downloaded_filename_re from viofosync_lib.cameras import CAMERA_LETTERS from ..db import Database @@ -186,27 +187,24 @@ def reconcile( def _camera_from_filename(filename: str) -> Optional[str]: # Handles both ``…_0001F.MP4`` and ``…_0001PF.MP4`` / # ``…_0001EF.MP4`` — the optional prefix letter encodes the - # event type (P=parking, E=event); the camera letter set - # comes from the registry. - import re as _re - m = _re.match( - rf"^\d{{4}}_\d{{4}}_\d{{6}}_\d+[PE]?([{CAMERA_LETTERS}])\.MP4$", - filename, - _re.IGNORECASE, - ) - return m.group(1).upper() if m else None + # event type (P=parking, E=event). Older Viofo filenames may omit + # the camera letter entirely; treat those as front-camera clips. + m = downloaded_filename_re.match(filename) + if not m: + return None + suffix = (m.group("camera") or "").upper() + if not suffix: + return "F" + camera = suffix[-1] + return camera if camera in CAMERA_LETTERS else None def _event_from_filename(filename: str) -> Optional[str]: - import re as _re - m = _re.match( - rf"^\d{{4}}_\d{{4}}_\d{{6}}_\d+([PE])?[{CAMERA_LETTERS}]\.MP4$", - filename, - _re.IGNORECASE, - ) + m = downloaded_filename_re.match(filename) if not m: return None - prefix = (m.group(1) or "").upper() + suffix = (m.group("camera") or "").upper() + prefix = suffix[-2] if len(suffix) >= 2 else "" return {"P": "parking", "E": "event"}.get(prefix, "normal") @@ -216,8 +214,11 @@ def _event_from_filename(filename: str) -> Optional[str]: # Filenames end in ``…NNNNN[PE]?[FRTI].MP4`` — the camera letter # is the character immediately before ``.MP4``, and the byte # before that is either a digit (normal) or P/E. -_CAM_SQL = "upper(substr(filename, -5, 1))" -_EVT_PREFIX_SQL = "upper(substr(filename, -6, 1))" +_CAM_SQL = "COALESCE(NULLIF(camera, ''), 'F')" +_EVT_PREFIX_SQL = ( + "CASE event_type WHEN 'parking' THEN 'P' " + "WHEN 'event' THEN 'E' ELSE '' END" +) def next_pending( @@ -444,13 +445,13 @@ def list_page( def _day_expr() -> str: """SQL expression for the YYYY-MM-DD day key derived from - the filename (``YYYY_MMDD_HHMMSS_NN.MP4``). Uses the - filename rather than ``recorded_at`` so grouping is - consistent even for rows missing a timestamp.""" + the recording timestamp. Falls back to filename slicing for + legacy rows that predate ``recorded_at``.""" return ( - "substr(filename,1,4) || '-' || " - "substr(filename,6,2) || '-' || " - "substr(filename,8,2)" + "CASE WHEN recorded_at IS NOT NULL " + "THEN strftime('%Y-%m-%d', recorded_at, 'unixepoch') " + "ELSE substr(filename,1,4) || '-' || " + "substr(filename,6,2) || '-' || substr(filename,8,2) END" ) @@ -477,7 +478,12 @@ def _kind_filters( both aliased and unaliased queries. """ prefix = f"{alias}." if alias else "" - evt = _EVT_PREFIX_SQL.replace("filename", f"{prefix}filename") + evt = ( + f"CASE {prefix}event_type " + "WHEN 'parking' THEN 'P' " + "WHEN 'event' THEN 'E' " + "ELSE '' END" + ) ro_expr = _RO_SQL.replace("source_dir", f"{prefix}source_dir") if driving and parking and ro: @@ -581,8 +587,13 @@ def list_day_items( params.extend(kind_params) where = "WHERE " + " AND ".join(clauses) - cam_dq = _CAM_SQL.replace("filename", "dq.filename") - evt_dq = _EVT_PREFIX_SQL.replace("filename", "dq.filename") + cam_dq = "COALESCE(NULLIF(dq.camera, ''), 'F')" + evt_dq = ( + "CASE dq.event_type " + "WHEN 'parking' THEN 'P' " + "WHEN 'event' THEN 'E' " + "ELSE '' END" + ) ro_dq = _RO_SQL.replace("source_dir", "dq.source_dir") with db.conn() as c: diff --git a/web/services/scanner.py b/web/services/scanner.py index 1f97114..d22d0b0 100644 --- a/web/services/scanner.py +++ b/web/services/scanner.py @@ -91,13 +91,14 @@ def _clip_meta_for( if not os.path.isfile(path): return None - camera_field = m.group("camera") + camera_field = (m.group("camera") or "F").upper() + camera = camera_field[-1] if camera_field[-1:] in "FRTI" else "F" return ClipMeta( path=path, basename=filename, group_name=ts.strftime("%Y-%m-%d"), # always daily key in UI timestamp=ts, - camera=camera_field.upper(), + camera=camera, sequence=int(m.group("sequence")), event_type=_event_type_for(camera_field, source_dir), size_bytes=os.path.getsize(path), @@ -270,4 +271,3 @@ def scan(db: Database, destination: str, grouping: str, hub=None, loop=None) -> _dq.enqueue_missing(db, priority=1, now=int(time.time())) return len(seen_paths) - From 0af5a4ce64dd5d7f157c5e0d14fcb6ca96a77dd8 Mon Sep 17 00:00:00 2001 From: Brian Hynds Date: Thu, 25 Jun 2026 19:25:15 -0400 Subject: [PATCH 2/3] Fix compact filename PR checks --- tests/test_cameras.py | 6 +++++- tests/test_queue_day_grouping.py | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_cameras.py b/tests/test_cameras.py index 116d846..6051ca0 100644 --- a/tests/test_cameras.py +++ b/tests/test_cameras.py @@ -8,6 +8,8 @@ """ from __future__ import annotations +import fnmatch + from viofosync_lib._archive import downloaded_filename_glob from viofosync_lib.cameras import ( CAMERA_LETTERS, @@ -52,7 +54,9 @@ def test_pair_slot_of_rear_fallback(): def test_glob_derivation(): - assert downloaded_filename_glob.endswith(f"_*[{CAMERA_LETTERS}].MP4") + assert fnmatch.fnmatch("2026_0625_171242_0001F.MP4", downloaded_filename_glob) + assert fnmatch.fnmatch("20260625171242_000086.MP4", downloaded_filename_glob) + assert not fnmatch.fnmatch("20260625171242_000086.JPG", downloaded_filename_glob) def test_js_mirror_matches_registry(): diff --git a/tests/test_queue_day_grouping.py b/tests/test_queue_day_grouping.py index 2a228e5..7c5a1b0 100644 --- a/tests/test_queue_day_grouping.py +++ b/tests/test_queue_day_grouping.py @@ -10,9 +10,7 @@ def _recorded_at() -> int: return int( - dt.datetime( - 2026, 6, 25, 17, 12, 42, tzinfo=dt.timezone.utc - ).timestamp() + dt.datetime(2026, 6, 25, 17, 12, 42, tzinfo=dt.UTC).timestamp() ) From a142296b9d89da27c45c737c9758e3533369eca2 Mon Sep 17 00:00:00 2001 From: Brian Hynds Date: Thu, 25 Jun 2026 19:33:54 -0400 Subject: [PATCH 3/3] Use filename dates for queue grouping --- tests/test_queue_day_grouping.py | 34 +++++++++++++++++++++++++++++--- web/services/queue.py | 16 +++++++++++---- web/static/app.js | 15 ++++++++------ 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/tests/test_queue_day_grouping.py b/tests/test_queue_day_grouping.py index 7c5a1b0..6f4fe2c 100644 --- a/tests/test_queue_day_grouping.py +++ b/tests/test_queue_day_grouping.py @@ -8,13 +8,13 @@ from web.services import queue -def _recorded_at() -> int: +def _recorded_at(day: int = 25) -> int: return int( - dt.datetime(2026, 6, 25, 17, 12, 42, tzinfo=dt.UTC).timestamp() + dt.datetime(2026, 6, day, 17, 12, 42, tzinfo=dt.UTC).timestamp() ) -def test_compact_filename_uses_recorded_at_for_day_grouping(tmp_path): +def test_compact_filename_uses_filename_date_for_day_grouping(tmp_path): db = Database(str(tmp_path / "test.db")) now = int(time.time()) with db.write() as c: @@ -44,3 +44,31 @@ def test_compact_filename_uses_recorded_at_for_day_grouping(tmp_path): assert len(items) == 1 assert items[0]["kind_camera"] == "F" assert items[0]["kind_event"] == "normal" + + +def test_compact_filename_day_grouping_ignores_timestamp_timezone_shift(tmp_path): + db = Database(str(tmp_path / "test.db")) + now = int(time.time()) + with db.write() as c: + c.execute( + """ + INSERT INTO download_queue ( + filename, source_dir, remote_size, recorded_at, + camera, event_type, state, enqueued_at + ) VALUES (?,?,?,?,?,?,?,?) + """, + ( + "20260625171242_000086.MP4", + "/DCIM/Movie", + 1024, + _recorded_at(day=26), + "F", + "normal", + "pending", + now, + ), + ) + + assert [d["day"] for d in queue.list_days(db)] == ["2026-06-25"] + assert len(queue.list_day_items(db, "2026-06-25")) == 1 + assert queue.list_day_items(db, "2026-06-26") == [] diff --git a/web/services/queue.py b/web/services/queue.py index 87e4502..f9d11cc 100644 --- a/web/services/queue.py +++ b/web/services/queue.py @@ -444,11 +444,19 @@ def list_page( def _day_expr() -> str: - """SQL expression for the YYYY-MM-DD day key derived from - the recording timestamp. Falls back to filename slicing for - legacy rows that predate ``recorded_at``.""" + """SQL expression for the YYYY-MM-DD day key. + + Prefer the dashcam filename date so day grouping stays independent of + server/browser timezone. Fall back to recorded_at for legacy rows whose + filenames cannot be parsed by either Viofo naming style. + """ return ( - "CASE WHEN recorded_at IS NOT NULL " + "CASE " + "WHEN filename GLOB '[0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_*' " + "THEN substr(filename,1,4) || '-' || substr(filename,6,2) || '-' || substr(filename,8,2) " + "WHEN filename GLOB '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]*_*' " + "THEN substr(filename,1,4) || '-' || substr(filename,5,2) || '-' || substr(filename,7,2) " + "WHEN recorded_at IS NOT NULL " "THEN strftime('%Y-%m-%d', recorded_at, 'unixepoch') " "ELSE substr(filename,1,4) || '-' || " "substr(filename,6,2) || '-' || substr(filename,8,2) END" diff --git a/web/static/app.js b/web/static/app.js index 4fa779f..207277e 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -1848,13 +1848,16 @@ function renderPagination(total) { // ---------- Downloads ---------- // Two-digit hour ("00".."23") for a queue item, derived from the -// dashcam filename (YYYY_MMDD_HHMMSS_…). Filename-derived (not -// recorded_at) to stay timezone-stable and consistent with the -// server's day grouping (_day_expr in services/queue.py). Names that -// don't match bucket under "??" so they stay visible and sort last. +// dashcam filename. Supports both YYYY_MMDD_HHMMSS_... and compact +// YYYYMMDDHHMMSS_... names. Filename-derived (not recorded_at) to stay +// timezone-stable and consistent with the server's day grouping. +// Names that don't match bucket under "??" so they stay visible and sort last. function hourKeyForItem(it) { - const m = /^\d{4}_\d{4}_(\d{2})/.exec(it.filename || ""); - return m ? m[1] : "??"; + const name = it.filename || ""; + const separated = /^\d{4}_\d{4}_(\d{2})/.exec(name); + if (separated) return separated[1]; + const compact = /^\d{8}(\d{2})\d{4}_/.exec(name); + return compact ? compact[1] : "??"; } // Bucket a day's items by hour. Returns [{ hour, items }] with hours