diff --git a/CHANGELOG.md b/CHANGELOG.md index c557382..571d692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## v2.3 — 2026-06-10 + +### Added + +#### Timeline Video Editor + +A multi-camera timeline editor for the archive: scrub the front/rear channels on a shared playhead with per-clip filmstrips, set in/out trim points, and cut between cameras to export a single switched-angle video. Includes frame-accurate keyboard shortcuts. Join, picture-in-picture, and switched exports are now hardware-accelerated via Intel QuickSync where available, with VAAPI and software fallbacks. + +#### Export Jobs + +- Animated filmstrip preview on each job — hover to scrub through the finished video. +- Click a job's thumbnail to play the export in the viewer. +- Output **Length** and **Size** columns in the jobs table. +- Switched-camera exports now carry one continuous front-camera audio track, removing the audible jump at each camera switch. + +### Changed + +- Base image moved to Debian + jellyfin-ffmpeg to unlock QuickSync on Intel iGPUs; VAAPI and software remain as fallbacks, so a host without a working iGPU degrades transparently. +- Archive updates live on a clip-indexed push instead of per-client polling, and the per-day GPS route aggregation is cached. +- UI polish: background dither, clearer labels, and archive view state that persists across navigation. + +### Fixed + +A broad reliability and security hardening pass (full per-item detail in `CLAUDE.md`): + +- **Worker lifecycle** — sync and export workers now shut down within a bounded timeout, and an in-flight or paused ffmpeg export is no longer left running after stop. Changing the dashcam address or toggling scheduled sync starts and stops the worker at runtime, without a restart. +- **Responsiveness** — NAS directory walks, SQLite transactions, and the quota disk-usage scan run off the event loop, so a slow or busy recordings volume no longer freezes the UI and live updates. +- **Data safety** — manual-import staging recovers completed clips after a crash instead of deleting them; downloads that fail their size check are rejected rather than archived truncated; corrupt or truncated MP4s no longer spin a worker at 100% CPU; and partial thumbnails, filmstrips, or exports can't be served from cache or left to count against the quota. +- **Disk full** — a full recordings volume raises a sticky "disk full" sync error and pauses the queue, instead of marking every clip failed. +- **MQTT** — reconnect backoff resets after a stable connection, retained discovery configs are cleaned up on the first node-id/prefix change, and timed-out probes are reaped. +- **Security** — clip filenames and geocoded place names are HTML-escaped before display; the live-events WebSocket rejects cross-origin handshakes; the MQTT broker password is no longer returned by the settings API; and retention will not delete a clip while an export is reading it. +- Queue counters update immediately after the Prioritise and Retry actions. + ## v2.2 — 2026-06-06 ### Added diff --git a/Dockerfile b/Dockerfile index 1935223..fe198d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,53 @@ -FROM alpine:3.23 +FROM python:3.12-slim-bookworm LABEL maintainer="Rob Smith https://github.com/RobXYZ" # TARGETARCH is set automatically by `docker buildx build` (amd64, -# arm64, …). Plain `docker build` does NOT set it; in that case we -# fall back to `apk --print-arch`, which reports the actual -# container architecture (x86_64, aarch64, …) and is correct under -# both native builds and QEMU emulation. +# arm64, …). Plain `docker build` does NOT set it; fall back to dpkg. ARG TARGETARCH # System deps: -# - python3 + pip: runtime + installing web deps -# - ffmpeg: exports + thumbnails -# - bash, shadow, tzdata: entrypoint + PUID/PGID remapping -# - intel-media-driver, libva-utils (Intel x86_64 only): VA-API -# userspace + diagnostic tool. ffmpeg's h264_qsv / h264_vaapi -# need iHD_drv_video.so to talk to an Intel iGPU when the host -# maps /dev/dri into the container; without it the MFX runtime -# fails immediately with "MFX session: -9". `vainfo` from -# libva-utils is a one-liner diagnostic the operator can run via -# `docker exec` to verify the passthrough is wired up correctly. -# These packages don't exist on linux/arm64. The app's encoder -# probe (web/services/exporter.py) runtime-tests every candidate -# and falls back to libx264 software encode if QSV / VAAPI -# aren't available, so the missing packages on ARM degrade -# transparently. -RUN apk add --no-cache \ - bash python3 py3-pip ffmpeg shadow su-exec tzdata && \ - arch="${TARGETARCH:-$(apk --print-arch)}" && \ +# - python is in the base image; pip installs web deps (PEP 668 -> +# --break-system-packages, safe in a container). +# - jellyfin-ffmpeg7: exports + thumbnails. Unlike Debian's stock +# ffmpeg (and unlike Alpine's musl build), the Jellyfin bundle ships +# the *legacy* Intel Media SDK runtime alongside oneVPL, which is what +# the DS920+'s Gen-9.5 (Gemini Lake) iGPU needs for QuickSync. Stock +# runtimes only drive Gen 12+, failing Gen 9.5 with "MFX session: -9"; +# the bundle is exactly how Jellyfin solved QSV-in-Docker. It also +# bundles the iHD VAAPI driver, so VAAPI keeps working as a fallback. +# The binary installs to /usr/lib/jellyfin-ffmpeg/{ffmpeg,ffprobe}; +# we symlink it onto PATH so shutil.which("ffmpeg") finds it unchanged. +# - gosu: privilege drop in entrypoint.sh (Debian's su-exec equivalent; +# same initgroups() semantics so the GPU render-group logic in +# setuid.sh keeps working). +# - vainfo (amd64): a one-line passthrough diagnostic. (On Debian the +# binary ships in the `vainfo` package, not Alpine's `libva-utils`.) +# On arm64 jellyfin-ffmpeg installs too but QSV simply won't probe-pass; +# exports degrade to software/VAAPI exactly as before. The app's encoder +# probe (web/services/exporter.py) runtime-tests every candidate and +# falls back to libx264, so a host without a working iGPU degrades +# transparently. +RUN set -eux; \ + arch="${TARGETARCH:-$(dpkg --print-architecture)}"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + bash ca-certificates gnupg gosu tzdata; \ + install -d /etc/apt/keyrings; \ + gpg_url="https://repo.jellyfin.org/jellyfin_team.gpg.key"; \ + apt-get install -y --no-install-recommends curl; \ + curl -fsSL "$gpg_url" | gpg --dearmor -o /etc/apt/keyrings/jellyfin.gpg; \ + echo "deb [signed-by=/etc/apt/keyrings/jellyfin.gpg] https://repo.jellyfin.org/debian bookworm main" \ + > /etc/apt/sources.list.d/jellyfin.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends jellyfin-ffmpeg7; \ case "$arch" in \ - amd64|x86_64) apk add --no-cache intel-media-driver libva-utils ;; \ - esac && \ + amd64) apt-get install -y --no-install-recommends vainfo ;; \ + esac; \ + ln -sf /usr/lib/jellyfin-ffmpeg/ffmpeg /usr/local/bin/ffmpeg; \ + ln -sf /usr/lib/jellyfin-ffmpeg/ffprobe /usr/local/bin/ffprobe; \ + apt-get purge -y curl gnupg; \ + apt-get autoremove -y; \ + rm -rf /var/lib/apt/lists/*; \ useradd -UMr dashcam COPY LICENSE / @@ -40,9 +58,9 @@ ENV PUID="" \ PGID="" \ RECORDINGS="/recordings" -# Install Python deps into the system site-packages. Alpine's -# pip refuses by default (PEP 668); --break-system-packages is -# safe inside a container. +# Install Python deps into the system site-packages. pip refuses +# by default (PEP 668 on Debian Bookworm+); --break-system-packages +# is safe inside a container. COPY requirements.txt /requirements.txt RUN pip install --no-cache-dir --break-system-packages \ -r /requirements.txt diff --git a/README.md b/README.md index 58e1c99..210618e 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,47 @@ # viofosync -Self-hosted web app for syncing, browsing, and exporting recordings from a Viofo dashcam (tested with the A229 Pro) over Wi-Fi. Runs as a single Docker container on a NAS or any always-on host on the same network as the dashcam. +![CI](https://github.com/RobXYZ/viofosync/actions/workflows/ci.yml/badge.svg) ![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg) ![Docker](https://img.shields.io/docker/pulls/robxyz/viofosync) -> **v2 is a full rewrite.** v1 was a cron-driven CLI based on [BlackVueSync](https://github.com/acolomba/BlackVueSync). v2 uses the same dashcam protocol but ships a web UI, journey-detected GPS maps, ffmpeg exports, JSON-backed settings, a first-run setup wizard, and a UI-driven download manager. The v1 cron CLI is preserved on the `main` branch. +Self-hosted web app for syncing, browsing, and exporting recordings from a Viofo dashcam (tested with the A229 Pro) over Wi-Fi. Runs as a single Docker container on a NAS or any always-on host on the same network as the dashcam. -![Download manager](screenshots/download_manager.png) +> **v2 is a full rewrite.** v1 was a cron-driven CLI based on [BlackVueSync](https://github.com/acolomba/BlackVueSync). v2 uses the same dashcam protocol but ships a web UI, journey-detected GPS maps, a timeline video editor, ffmpeg exports, JSON-backed settings, a first-run setup wizard, and a UI-driven download manager. The v1 cron CLI is preserved on the `main` branch. ## Features -- **Archive browser** — clips grouped by day, paired front/rear, in-browser playback. -- **GPS journeys** — clickable map per trip with auto-split stops and reverse-geocoded place names. -- **Exports** — original clips, joined front/rear or picture-in-picture videos via ffmpeg. -- **Download manager** — live progress with session speed and ETA, a reorderable queue. -- **Auto-delete from dashcam** *(optional)* — frees SD card space once a clip is safely downloaded. -- **Settings page** — runtime settings hot-reload; no Docker env vars to fiddle with. -- **Home Assistant support** — auto-discovered sensors and buttons via MQTT. +- **Automatic Wi-Fi sync** - clips copy from the dashcam in your car when it joins your home wi-fi. +- **Archive browser** - clips grouped by day, played in your browser. Nothing to install on your phone or laptop. +- **Journey maps** - automatic journey detection with each trip shown on a map with stops detected and place names looked up automatically. +- **Video editor** - trim clips and cut between the front and rear cameras, then export a single video. +- **Flexible exports** - original clips, joined front/rear, picture-in-picture, or edited cuts; hardware-accelerated where your system supports it. +- **Storage management** - set a size or age limit and the oldest footage is pruned to fit; optional auto-delete clears the camera's SD card once a clip is safely saved. +- **Easy browser-based setup** - a first-run wizard, then a settings page. +- **Home Assistant support** - over MQTT, with sync status, alerts, and action buttons. + +![Timeline video editor](screenshots/timeline_editor.webp)![Download manager](screenshots/download_manager.webp) -## Hardware +## Contents -The dashcam must stay powered on and connected to Wi-Fi. A hardwire kit (e.g. Viofo HK4) plus a dedicated dashcam battery is recommended. +- [Features](#features) +- [Getting started](#getting-started) +- [Configuration](#configuration) +- [Home Assistant](#home-assistant-via-mqtt) +- [Reference](#reference) +- [About](#about) -It should join your LAN in Wi-Fi **station** mode. As of May 2026 the official A229 Pro firmware does not retain Wi-Fi state across reboots but Viofo support will provide a custom firmware on request. +## Getting started -Reserve the dashcam's IP on your router so it doesn't change. +### Requirements -## Quick start +> [!NOTE] +> #### Most users will need +> - **Viofo Wi-Fi dashcam** connected to your LAN in station mode +> - **Viofo special firmware** to keep station mode always-on (supplied by Viofo support on request) +> - **Hardwire kit** (Viofo HK4) to keep the camera powered when parked - a dedicated dashcam battery is recommended for extended downloads +> - **Reserved IP** for the dashcam on your router, so it doesn't change +> - **NAS or always-on host** with large storage that can run Docker +> - **Optional: hardware video encoder + fast LAN** - recommended for the video editing features + +### Quick start ```bash docker run -d \ @@ -39,12 +56,36 @@ docker run -d \ robxyz/viofosync ``` +Or use the included `[docker-compose.yml](docker-compose.yml)`, which has the same settings plus a commented-out GPU passthrough block (see below). + Open `http://:8080` and the first boot redirects you to a one-screen setup wizard at `/setup`. Enter the dashcam IP and an admin password (12+ characters) to finish. The wizard writes `/config/config.json` with a freshly-generated `SESSION_SECRET` and a bcrypt hash of the password — neither is held in env vars or the image. After setup, every other setting lives on the **Settings** page in the UI. > ⚠ **Setup window safety.** Until the wizard is submitted there is no auth on the container — the wizard self-disables after first submission and the route returns 404 thereafter. Don't expose the container to the public internet during this window. +### Hardware-accelerated exports + +Exports (join, picture-in-picture, switched) and thumbnails use ffmpeg. At startup the app probes the host's encoders — QuickSync (Intel iGPU), VAAPI, NVENC, VideoToolbox — and falls back to software (libx264) if none work, so exports always run. + +To use an Intel iGPU, pass the render node through: + +```bash +docker run ... --device /dev/dri:/dev/dri robxyz/viofosync +``` + +The entrypoint auto-detects the render node's group and adds the app user to it. Some hosts (notably Synology DSM) need the group granted explicitly — find the GID and add it: + +```bash +docker exec sh -c 'stat -c %g /dev/dri/renderD128' # often 937 on Synology +# docker run ... --group-add 937 (or group_add: ["937"] in docker-compose.yml) +``` + +Confirm it engaged with `docker exec vainfo`, and check the startup log for `export encoder available: … qsv …`. + +> [!NOTE] +> On arm64 hosts QuickSync won't probe-pass; exports degrade to VAAPI or software automatically. + ## Configuration The only Docker-level env vars are: @@ -58,7 +99,7 @@ App-level settings (sync interval, dashcam IP, encoder, geocoding email, web por ### Importing without Wi-Fi -Use **Import manually** in the web UI to ingest clips you already have on disk. Two modes: +Use **Import manually** in the web UI to ingest clips you already have on disk or the SD card. Two modes: - **Upload** — pick a folder in your browser; clips upload one at a time and slot straight into the archive. On a quota-bound archive it makes room as it goes, evicting the oldest clips (never anything newer than what you're importing). - **Folder** — copy clips into the `import` folder inside your recordings share, then **Scan** → **Ingest**. By default this is `recordings/import`; for a one-off import from a different path, type it in the Import dialog's Folder tab, or set a persistent default via the advanced `IMPORT_PATH` key in `/config/config.json`. @@ -74,17 +115,15 @@ The source is only ever **read** — originals on the card/USB are never deleted Imported clips are recognised by Viofo naming (`YYYY_MMDD_HHMMSS_NNNN[event][cam].MP4`); locked clips under an `RO/` folder keep their protected status. Non-matching files are left untouched. -## Alternative camera address +### Alternative camera address -You can set an optional **Alternative address** (Settings → Dashcam) — a second IP/host for the **same** dashcam. It is **not** for a second camera. +You can set an optional **Alternative address** (Settings → Dashcam) for the **same** dashcam. -This is for reaching one camera at more than one address depending on where the car is, for example: +This can be useful for reaching the camera on a second network: -- A Raspberry Pi running a VPN hotspot, so you can reach the dashcam remotely when the car is away from home. +- A Raspberry Pi running a VPN hotspot in the car, so you can reach the dashcam remotely. - A site-to-site VPN to a second location the car is regularly parked at, where the camera sits on a different subnet/IP. -The alternative uses the same form as the primary (IP or hostname, plain `http`, port 80). - ## Home Assistant via MQTT viofosync can publish state and accept actions over MQTT, with full Home Assistant auto-discovery. @@ -115,15 +154,17 @@ For "prioritize the last N hours", publish to `{node_id}/cmd/prioritize_recent` - The MQTT password is stored in `config.json` in plaintext, alongside the bcrypt hash of the admin password and the session secret. The same access controls already apply to that file. -## Reverse geocoding +## Reference + +### Reverse geocoding Journey and stop cards display their start/end as *"Street, Town"* via Nominatim (OpenStreetMap). Lookups are rate-limited to 1/second per [Nominatim's usage policy](https://operations.osmfoundation.org/policies/nominatim/) and cached in the `geocode_cache` table (coords rounded to 3 d.p., ≈110 m). Set **Nominatim email** in Settings → GPS & Geocoding to identify your install per OSM's terms; toggle the **GPS maps** filter off on the Archive page to skip the Leaflet + Nominatim machinery entirely for low-bandwidth browsing. -## XML vs HTML listing +### XML vs HTML listing By default the app scrapes the dashcam's HTML directory listings (`/DCIM/Movie`, `/DCIM/Movie/Parking`, `/DCIM/Movie/RO`), which is noticeably faster on some firmware than the XML API (`/?custom=1&cmd=3015&par=1`). Toggle off **Use HTML directory listing** in Settings → Dashcam to fall back to XML. -## Migrating from v1 +### Migrating from v1 Existing installs with a `viofosync.env` file are migrated automatically on first boot of the v2 image: @@ -133,7 +174,7 @@ Existing installs with a `viofosync.env` file are migrated automatically on firs `PUID` / `PGID` / `TZ` env vars work the same as v1. -## Running without Docker +### Running without Docker For development or for hosts that don't have Docker: @@ -145,16 +186,18 @@ CONFIG_DIR=/path/to/config RECORDINGS=/path/to/archive \ `web.launcher` reads `WEB_HOST` / `WEB_PORT` from `config.json` (defaults `0.0.0.0:8080`) and re-execs into uvicorn. On first run, browse to `http://localhost:8080/setup`. `ffmpeg` must be on `$PATH` for thumbnails and exports. -## AI Code +## About + +### AI Code -This opensource project uses AI generated code and is intended for personal home use. It is not recommended that the server is exposed to the public internet. +viofosync is an open-source project built with substantial AI assistance, intended for personal use on a home network. Its security model assumes a trusted LAN - a single password over plain HTTP - so keep it behind your network or a VPN rather than exposing it directly to the public internet. -## Credits +### Credits The GPX extraction logic uses the method described at [https://sergei.nz/extracting-gps-data-from-viofo-a119-and-other-novatek-powered-cameras/](https://sergei.nz/extracting-gps-data-from-viofo-a119-and-other-novatek-powered-cameras/). This software is unaffiliated with Viofo or any other vendor. -## License +### License MIT — see [LICENSE](LICENSE). diff --git a/docker-compose.yml b/docker-compose.yml index fb4795b..f7c7202 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,23 @@ services: ports: - "8080:8080" + # Optional: pass through the GPU for hardware-accelerated decode/encode + # (exports). On an Intel iGPU this now uses QuickSync (QSV) first and + # falls back to VA-API, then software. Uncomment on a NAS with an Intel + # iGPU. Verify it engaged with: docker exec vainfo and check the + # startup log line "export encoder available: … qsv …". + # devices: + # - /dev/dri:/dev/dri + # + # The entrypoint auto-detects the render node's group and adds the app + # user to it, so this is usually all you need. Some hosts (notably + # Synology DSM) still require the render group to be granted explicitly — + # if hardware accel doesn't engage, find the GID with + # docker exec sh -c 'stat -c %g /dev/dri/renderD128' + # and add it here (937 is common on Synology): + # group_add: + # - "937" # GID owning /dev/dri/renderD128 — the iGPU render node + volumes: # Config directory. A template viofosync.env is seeded here on first # run — edit it (ADDRESS, WEB_PASSWORD, etc.) and restart to apply. diff --git a/entrypoint.sh b/entrypoint.sh index 942e197..0bf7cf5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,4 +4,8 @@ set -e mkdir -p /config /recordings /setuid.sh -exec su-exec dashcam:dashcam python3 -m web.launcher +# `gosu dashcam` (user form, NOT `dashcam:dashcam`): the user form calls +# initgroups() so the supplementary groups from /etc/group — including the GPU +# render group added in setuid.sh — are applied. The explicit `user:group` form +# would replace them with just that one group, dropping render-node access. +exec gosu dashcam python3 -m web.launcher diff --git a/screenshots/download_manager.png b/screenshots/download_manager.png deleted file mode 100644 index bd06bd5..0000000 Binary files a/screenshots/download_manager.png and /dev/null differ diff --git a/screenshots/download_manager.webp b/screenshots/download_manager.webp new file mode 100644 index 0000000..621a843 Binary files /dev/null and b/screenshots/download_manager.webp differ diff --git a/screenshots/timeline_editor.webp b/screenshots/timeline_editor.webp new file mode 100644 index 0000000..cbe4f56 Binary files /dev/null and b/screenshots/timeline_editor.webp differ diff --git a/scripts/bench_filmstrip.sh b/scripts/bench_filmstrip.sh new file mode 100755 index 0000000..6671ec6 --- /dev/null +++ b/scripts/bench_filmstrip.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# +# bench_filmstrip.sh — measure filmstrip-sprite decode cost on THIS host. +# +# Why this exists: viofosync runs on many NAS models (Synology Celeron/Atom, +# Ryzen embedded, ARM, …). Whether hardware decode (-hwaccel) beats software +# for filmstrip generation depends entirely on the box's CPU and iGPU, so the +# only trustworthy data comes from running the real command on the real +# hardware with a real 4K dashcam clip. Apple-Silicon dev-machine numbers do +# not transfer. +# +# It runs the EXACT command web/services/filmstrip.py issues, in several decode +# configurations, and reports for each: +# real wall-clock seconds (lower = faster for one clip) +# cpu user+sys seconds (lower = more decode offloaded off the CPU — +# THIS is the number that matters on a weak NAS +# that spikes under concurrent filmstrip jobs) +# dims sprite WxH, to confirm the output is correct (not garbage) +# +# Usage: +# ./bench_filmstrip.sh /path/to/4k_clip.MP4 [output_dir] +# +# ffmpeg/ffprobe are taken from PATH; override with env vars if needed, e.g. +# on Synology where ffmpeg ships inside a package: +# FFMPEG=/var/packages/ffmpeg/target/bin/ffmpeg \ +# FFPROBE=/var/packages/ffmpeg/target/bin/ffprobe \ +# ./bench_filmstrip.sh /volume1/dashcam/2024_0101_120000_0001F.MP4 +# +set -u +export LC_ALL=C # force '.' decimal separator for time/ffprobe parsing + +# --- production constants (keep in sync with web/services/filmstrip.py) --- +INTERVAL_S=8 +TILE_W=160 +TILE_H=90 +RUNS="${RUNS:-3}" # runs per config (keep best); set RUNS=1 for a fast first pass + +FFMPEG="${FFMPEG:-$(command -v ffmpeg || true)}" +FFPROBE="${FFPROBE:-$(command -v ffprobe || true)}" + +die() { echo "error: $*" >&2; exit 1; } + +[ -n "$FFMPEG" ] || die "ffmpeg not found (set FFMPEG=/path/to/ffmpeg)" +[ -n "$FFPROBE" ] || die "ffprobe not found (set FFPROBE=/path/to/ffprobe)" + +CLIP="${1:-}" +[ -n "$CLIP" ] || die "usage: $0 /path/to/4k_clip.MP4 [output_dir]" +[ -f "$CLIP" ] || die "clip not found: $CLIP" + +OUTDIR="${2:-$(dirname "$CLIP")/.bench_filmstrip}" +mkdir -p "$OUTDIR" || die "cannot create output dir: $OUTDIR" + +# --- describe the clip and compute the tile count the real code would use --- +# Probe each field separately: with -show_entries, ffprobe emits values in the +# stream's own field order (codec_name often precedes width/height), so a +# single combined query would mislabel them. +_probe1() { + "$FFPROBE" -v error -select_streams v:0 \ + -show_entries "stream=$1" -of default=noprint_wrappers=1:nokey=1 "$CLIP" +} +WIDTH=$(_probe1 width) +HEIGHT=$(_probe1 height) +CODEC=$(_probe1 codec_name) +DURATION=$("$FFPROBE" -v error -show_entries format=duration \ + -of default=noprint_wrappers=1:nokey=1 "$CLIP") +BITRATE=$("$FFPROBE" -v error -show_entries format=bit_rate \ + -of default=noprint_wrappers=1:nokey=1 "$CLIP" 2>/dev/null) + +# tiles = max(1, ceil(duration / INTERVAL_S)) — same as filmstrip.frame_count +TILES=$(awk -v d="$DURATION" -v i="$INTERVAL_S" \ + 'BEGIN{ n=int((d+i-1)/i); if(n<1)n=1; print n }') +VF="fps=1/${INTERVAL_S},scale=${TILE_W}:${TILE_H},tile=${TILES}x1" +EXPECT_W=$(( TILES * TILE_W )) + +echo "host : $(uname -srm)" +echo "ffmpeg : $FFMPEG" +echo "clip : $CLIP" +printf "video : %sx%s %s dur=%.0fs bitrate=%sk -> %d tiles (expect %dx%d sprite)\n" \ + "$WIDTH" "$HEIGHT" "$CODEC" "$DURATION" \ + "$(awk -v b="${BITRATE:-0}" 'BEGIN{printf "%.0f", b/1000}')" \ + "$TILES" "$EXPECT_W" "$TILE_H" +echo + +# --- which hwaccels did this ffmpeg build advertise? --- +HWACCELS=$("$FFMPEG" -hide_banner -hwaccels 2>/dev/null | tail -n +2 | tr -d ' ') +echo "advertised hwaccels: $(echo "$HWACCELS" | paste -sd',' -)" +[ -e /dev/dri/renderD128 ] && echo "found /dev/dri/renderD128 (iGPU render node present)" +echo + +printf "%-26s %8s %8s %-10s %s\n" "config" "real(s)" "cpu(s)" "dims" "status" +printf "%-26s %8s %8s %-10s %s\n" "--------------------------" "-------" "------" "----------" "------" + +# run_cfg LABEL +# Times the production command RUNS times, keeps the best wall-clock, and +# verifies the sprite dimensions. +run_cfg() { + label="$1"; shift + out="$OUTDIR/${label//[^A-Za-z0-9]/_}.jpg" + errlog="$OUTDIR/${label//[^A-Za-z0-9]/_}.err" + + best_real=""; best_cpu=""; rc=1 + for _ in $(seq 1 "$RUNS"); do + rm -f "$out" + TIMEFORMAT='%R %U %S' + # ffmpeg's own stderr -> errlog, stdout -> /dev/null; the `time` builtin's + # report is the only thing left on the compound's stderr, captured here. + t=$( { time "$FFMPEG" -loglevel error -y "$@" -i "$CLIP" -an \ + -vf "$VF" -frames:v 1 "$out" 2>"$errlog" 1>/dev/null ; } 2>&1 ) + rc=$? + [ $rc -eq 0 ] || break + real=$(echo "$t" | awk '{print $1}') + cpu=$(echo "$t" | awk '{printf "%.3f", $2+$3}') + if [ -z "$best_real" ] || awk -v a="$real" -v b="$best_real" 'BEGIN{exit !(a/dev/null | cut -c1-40) + printf "%-26s %8s %8s %-10s FAIL %s\n" "$label" "-" "-" "-" "$msg" + return 1 + fi + + dims=$("$FFPROBE" -v error -select_streams v:0 \ + -show_entries stream=width,height -of csv=p=0 "$out" 2>/dev/null) + status="ok" + case "$dims" in + "${EXPECT_W},${TILE_H}") status="ok" ;; + *) status="DIMS_MISMATCH" ;; + esac + printf "%-26s %8s %8s %-10s %s\n" "$label" "$best_real" "$best_cpu" "$dims" "$status" +} + +# Baseline + software keyframe-skip (the no-GPU path the code uses). +run_cfg "software (no skip)" +run_cfg "software+skip" + +# Every advertised hwaccel that's relevant to decode, with and without the +# keyframe skip — so you can see both the offload AND the surface-transfer +# cliff (hwaccel without skip downloads every decoded frame; can be ~25x). +for hw in videotoolbox cuda qsv vaapi; do + echo "$HWACCELS" | grep -qx "$hw" || continue + + run_cfg "hw:${hw}+skip" -hwaccel "$hw" -skip_frame nokey + run_cfg "hw:${hw} (no skip)" -hwaccel "$hw" + + # vaapi/qsv often need the render node named explicitly inside a container + # (the Synology /dev/dri passthrough case). If the bare form failed and the + # node exists, try again with the device so we learn what production needs. + if { [ "$hw" = "vaapi" ] || [ "$hw" = "qsv" ]; } && [ -e /dev/dri/renderD128 ]; then + run_cfg "hw:${hw}+skip+device" -hwaccel "$hw" \ + -hwaccel_device /dev/dri/renderD128 -skip_frame nokey + fi +done + +echo +echo "Read it like this:" +echo " * 'cpu(s)' is the number to watch on a NAS — lower means decode is" +echo " offloaded and the box stays responsive under concurrent jobs." +echo " * a big gap between a hwaccel's '+skip' and 'no skip' rows confirms the" +echo " surface-transfer cliff (skip_frame nokey is essential on every host)." +echo " * any DIMS_MISMATCH / FAIL row means that decode path is not usable here." +echo " * sprites left in: $OUTDIR (delete when done)" diff --git a/setuid.sh b/setuid.sh index 01178a9..4fe00b9 100755 --- a/setuid.sh +++ b/setuid.sh @@ -7,3 +7,34 @@ fi if [[ ${PGID:-0} -gt 0 ]]; then groupmod -o -g "$PGID" dashcam fi + +# Grant the app user access to the GPU render node(s) so hardware-accelerated +# decode/encode (filmstrips, exports) works. The render node is group-owned +# with no world access and its GID varies by NAS host. We gather candidate +# GIDs from two sources so either deployment style works: +# 1. the group that owns each /dev/dri node — automatic, no compose change +# 2. groups granted to the container via compose `group_add:` (the standard +# Synology approach; visible here as this script's own supplementary +# groups since it runs as root before the gosu drop) +# and add dashcam to each in /etc/group. This is required because entrypoint.sh +# drops privileges with `gosu dashcam`, whose initgroups() reads /etc/group +# — a group_add GID that isn't mirrored there would otherwise be lost. +# Best-effort: a failure here (e.g. no passthrough) must not stop startup. +gpu_gids="" +for dev in /dev/dri/renderD* /dev/dri/card*; do + [[ -e "$dev" ]] || continue + g=$(stat -c '%g' "$dev" 2>/dev/null) && gpu_gids="$gpu_gids $g" +done +gpu_gids="$gpu_gids $(id -G 2>/dev/null)" # group_add GIDs + +for gid in $gpu_gids; do + [[ "$gid" == 0 || "$gid" == "${PGID:-0}" ]] && continue # skip root / own primary + grp=$(awk -F: -v g="$gid" '$3 == g { print $1 }' /etc/group) + if [[ -z "$grp" ]]; then + grp="gpu_$gid" + groupadd -o -g "$gid" "$grp" 2>/dev/null || true + fi + usermod -aG "$grp" dashcam 2>/dev/null || true +done + +exit 0 diff --git a/tests/test_channel_of.py b/tests/test_channel_of.py new file mode 100644 index 0000000..3b21287 --- /dev/null +++ b/tests/test_channel_of.py @@ -0,0 +1,34 @@ +"""Tests for the camera -> timeline-channel mapping.""" +from __future__ import annotations + +from web.services import naming + + +def test_channel_of_front_variants(): + assert naming.channel_of("F") == "front" + assert naming.channel_of("PF") == "front" # parking front + assert naming.channel_of("EF") == "front" # event front + assert naming.channel_of("f") == "front" # case-insensitive + + +def test_channel_of_rear_variants(): + assert naming.channel_of("R") == "rear" + assert naming.channel_of("PR") == "rear" + + +def test_channel_of_interior(): + assert naming.channel_of("I") == "interior" + + +def test_channel_of_unknown_and_empty(): + assert naming.channel_of("") == "other" + assert naming.channel_of("X") == "other" + assert naming.channel_of(None) == "other" + + +def test_channel_order_and_labels(): + assert naming.CHANNEL_ORDER == ["front", "rear", "interior", "other"] + assert naming.CHANNEL_LABELS["front"] == "Front" + assert naming.CHANNEL_LABELS["rear"] == "Rear" + assert naming.CHANNEL_LABELS["interior"] == "Interior" + assert naming.CHANNEL_LABELS["other"] == "Other" diff --git a/tests/test_download_file_with_params.py b/tests/test_download_file_with_params.py new file mode 100644 index 0000000..776c430 --- /dev/null +++ b/tests/test_download_file_with_params.py @@ -0,0 +1,78 @@ +"""download_file_with must pass per-call attempts/timeout as +parameters, not by mutating module globals (which would let two +concurrent downloads clobber each other's settings).""" +from __future__ import annotations + +import datetime +from unittest.mock import patch + +import viofosync_lib as vfs +from viofosync_lib import _protocol +from viofosync_lib._archive import Recording + + +def _recording() -> Recording: + return Recording( + "2026_0101_120000_0001F.MP4", "/DCIM/Movie/x.MP4", 4, 0, + datetime.datetime(2026, 1, 1, 12, 0), 0, + ) + + +class _Resp: + def __init__(self, payload: bytes): + self._chunks = [payload] + + def read(self, n): + return self._chunks.pop(0) if self._chunks else b"" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_download_file_accepts_per_call_params(tmp_path): + seen = {} + + def fake_urlopen(url_or_req, *args, **kwargs): + if getattr(url_or_req, "get_method", lambda: "GET")() == "HEAD": + raise OSError("no HEAD") + seen["timeout"] = kwargs.get("timeout") + return _Resp(b"abcd") + + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + ok, _ = _protocol.download_file( + "http://192.0.2.1", _recording(), str(tmp_path), "", + max_attempts=4, socket_timeout=42.0, + ) + assert ok is True + assert seen["timeout"] == 42.0 + + +def test_download_file_with_does_not_mutate_globals_during_call(tmp_path): + """The override must reach urlopen by parameter while the module + globals stay at their defaults throughout the call.""" + default_timeout = _protocol.socket_timeout + default_attempts = _protocol.max_download_attempts + observed = {} + + def fake_urlopen(url_or_req, *args, **kwargs): + if getattr(url_or_req, "get_method", lambda: "GET")() == "HEAD": + raise OSError("no HEAD") + observed["call_timeout"] = kwargs.get("timeout") + observed["global_timeout_mid_call"] = _protocol.socket_timeout + observed["global_attempts_mid_call"] = _protocol.max_download_attempts + return _Resp(b"abcd") + + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + vfs.download_file_with( + "http://192.0.2.1", _recording(), str(tmp_path), "", + max_attempts=9, socket_timeout=99.0, + ) + + assert observed["call_timeout"] == 99.0 + assert observed["global_timeout_mid_call"] == default_timeout + assert observed["global_attempts_mid_call"] == default_attempts + assert _protocol.socket_timeout == default_timeout + assert _protocol.max_download_attempts == default_attempts diff --git a/tests/test_download_verify_fallback.py b/tests/test_download_verify_fallback.py new file mode 100644 index 0000000..e80a849 --- /dev/null +++ b/tests/test_download_verify_fallback.py @@ -0,0 +1,76 @@ +"""Truncated downloads must not be archived when HEAD fails. + +Some firmwares drop HEAD under load; verification used to be skipped +entirely then, so a connection closed cleanly mid-stream produced a +truncated file that was os.replace'd into the archive as a success. +The listing size (recording.size) is the fallback reference. +""" +from __future__ import annotations + +import datetime +import os +from unittest.mock import patch + +from viofosync_lib import _protocol +from viofosync_lib._archive import Recording + + +class _TruncatedResponse: + """Streams fewer bytes than the listing promised, then EOF — + exactly what a cleanly-closed truncated transfer looks like.""" + + def __init__(self, payload: bytes): + self._chunks = [payload] + + def read(self, n: int) -> bytes: + return self._chunks.pop(0) if self._chunks else b"" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _recording(size: int) -> Recording: + return Recording( + "2026_0101_120000_0001F.MP4", "/DCIM/Movie/x.MP4", + size, 0, datetime.datetime(2026, 1, 1, 12, 0, 0), 0, + ) + + +def test_truncated_download_rejected_when_head_fails(tmp_path, monkeypatch): + monkeypatch.setattr(_protocol, "max_download_attempts", 1) + + def fake_urlopen(url_or_req, *args, **kwargs): + # HEAD probe (Request object with method) fails; the GET + # (plain URL string) streams a truncated body. + if getattr(url_or_req, "get_method", lambda: "GET")() == "HEAD": + raise OSError("HEAD not supported under load") + return _TruncatedResponse(b"x" * 100) + + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + ok, _speed = _protocol.download_file( + "http://192.0.2.1", _recording(size=1000), str(tmp_path), "", + ) + + assert ok is False, "truncated download was reported as success" + dest = os.path.join(str(tmp_path), "2026_0101_120000_0001F.MP4") + assert not os.path.exists(dest), \ + "truncated file was moved into the archive" + + +def test_complete_download_succeeds_when_head_fails(tmp_path): + def fake_urlopen(url_or_req, *args, **kwargs): + if getattr(url_or_req, "get_method", lambda: "GET")() == "HEAD": + raise OSError("HEAD not supported under load") + return _TruncatedResponse(b"x" * 1000) # full size this time + + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + ok, _speed = _protocol.download_file( + "http://192.0.2.1", _recording(size=1000), str(tmp_path), "", + ) + + assert ok is True + dest = os.path.join(str(tmp_path), "2026_0101_120000_0001F.MP4") + assert os.path.getsize(dest) == 1000 diff --git a/tests/test_durations.py b/tests/test_durations.py new file mode 100644 index 0000000..75714df --- /dev/null +++ b/tests/test_durations.py @@ -0,0 +1,154 @@ +"""Tests for the clip duration ffprobe sweep.""" +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from web.db import Database +from web.services import durations + + +def _insert_clip(db, clip_id, path, duration_s=None): + with db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(id, path, basename, group_name, timestamp, camera, " + " sequence, event_type, has_gpx, gps_examined, scanned_at, duration_s) " + "VALUES (?,?,?,?,?,?,?,?,0,0,?,?)", + (clip_id, path, f"{clip_id}.MP4", "2026-06-02", + 1_717_312_440, "F", clip_id, "normal", 1_717_312_440, duration_s), + ) + + +async def test_probe_duration_parses_ffprobe(monkeypatch): + class _P: + async def communicate(self): + return (b"60.05\n", b"") + async def fake_exec(*a, **k): + return _P() + monkeypatch.setattr(durations.shutil, "which", lambda _n: "/usr/bin/ffprobe") + monkeypatch.setattr(durations.asyncio, "create_subprocess_exec", fake_exec) + assert await durations.probe_duration("/x.mp4") == pytest.approx(60.05) + + +async def test_probe_duration_none_without_ffprobe(monkeypatch): + monkeypatch.setattr(durations.shutil, "which", lambda _n: None) + assert await durations.probe_duration("/x.mp4") is None + + +async def test_sweep_updates_null_durations(tmp_path: Path, monkeypatch): + db = Database(str(tmp_path / "t.db")) + f1 = tmp_path / "clip1.mp4" + f1.write_bytes(b"\0") + f2 = tmp_path / "clip2.mp4" + f2.write_bytes(b"\0") + _insert_clip(db, 1, str(f1), duration_s=None) # needs probe + _insert_clip(db, 2, str(f2), duration_s=60.0) # already has one -> skipped + + async def fake_probe(path): + return 42.0, "mvhd" + monkeypatch.setattr(durations, "_probe_with_method", fake_probe) + + updated = await durations.sweep_missing_durations(db) + assert updated == 1 + with db.conn() as c: + d1 = c.execute("SELECT duration_s FROM clip_index WHERE id=1").fetchone()["duration_s"] + d2 = c.execute("SELECT duration_s FROM clip_index WHERE id=2").fetchone()["duration_s"] + assert d1 == pytest.approx(42.0) + assert d2 == pytest.approx(60.0) # untouched + + +async def test_sweep_skips_missing_files(tmp_path: Path, monkeypatch): + db = Database(str(tmp_path / "t.db")) + _insert_clip(db, 1, str(tmp_path / "gone.mp4"), duration_s=None) # file absent + async def fake_probe(path): + raise AssertionError("should not probe a missing file") + monkeypatch.setattr(durations, "_probe_with_method", fake_probe) + assert await durations.sweep_missing_durations(db) == 0 + + +async def test_sweep_persists_incrementally_when_interrupted( + tmp_path: Path, monkeypatch +): + """A sweep cancelled partway (e.g. server shutdown) must have already + persisted the clips it probed before the interruption — otherwise a + restart loses all progress and the sweep can never finish.""" + db = Database(str(tmp_path / "t.db")) + f1 = tmp_path / "clip1.mp4" + f1.write_bytes(b"\0") + f2 = tmp_path / "clip2.mp4" + f2.write_bytes(b"\0") + _insert_clip(db, 1, str(f1), duration_s=None) + _insert_clip(db, 2, str(f2), duration_s=None) + + async def fake_probe(path): + if path == str(f1): + return 42.0, "mvhd" + raise asyncio.CancelledError # shutdown hits while probing clip 2 + + monkeypatch.setattr(durations, "_probe_with_method", fake_probe) + + # concurrency=1 -> clip 1 is fully probed (and must be flushed) before + # clip 2 runs; batch_size=1 -> each result is persisted as it lands. + with pytest.raises(asyncio.CancelledError): + await durations.sweep_missing_durations(db, concurrency=1, batch_size=1) + + with db.conn() as c: + d1 = c.execute( + "SELECT duration_s FROM clip_index WHERE id=1" + ).fetchone()["duration_s"] + assert d1 == pytest.approx(42.0) # survived the interruption + + +async def test_sweep_logs_method_breakdown(tmp_path: Path, monkeypatch, caplog): + """The sweep reports how many clips it resolved via the fast mvhd path + vs the ffprobe fallback, so the fast path is visible in the Logs tab.""" + import logging + + db = Database(str(tmp_path / "t.db")) + for i in (1, 2, 3): + f = tmp_path / f"clip{i}.mp4" + f.write_bytes(b"\0") + _insert_clip(db, i, str(f), duration_s=None) + + async def fake(path): + # clip3 needs the ffprobe fallback; the rest resolve via mvhd + return (30.0, "ffprobe") if path.endswith("clip3.mp4") else (15.0, "mvhd") + monkeypatch.setattr(durations, "_probe_with_method", fake) + + with caplog.at_level(logging.INFO, logger="viofosync.durations"): + updated = await durations.sweep_missing_durations(db) + + assert updated == 3 + msgs = " ".join(r.getMessage() for r in caplog.records) + assert "2 via mvhd" in msgs + assert "1 via ffprobe" in msgs + + +async def test_sweep_writes_all_with_small_batches(tmp_path: Path, monkeypatch): + """Batched flushing must not drop rows: every probed clip is written + even when the batch size is smaller than the number of clips.""" + db = Database(str(tmp_path / "t.db")) + paths = [] + for i in range(1, 6): + f = tmp_path / f"clip{i}.mp4" + f.write_bytes(b"\0") + paths.append(str(f)) + _insert_clip(db, i, str(f), duration_s=None) + + async def fake_probe(path): + return 10.0, "mvhd" + + monkeypatch.setattr(durations, "_probe_with_method", fake_probe) + updated = await durations.sweep_missing_durations(db, batch_size=2) + assert updated == 5 + with db.conn() as c: + vals = [ + r["duration_s"] + for r in c.execute( + "SELECT duration_s FROM clip_index ORDER BY id" + ).fetchall() + ] + assert vals == [pytest.approx(10.0)] * 5 diff --git a/tests/test_export_control.py b/tests/test_export_control.py new file mode 100644 index 0000000..67acd4e --- /dev/null +++ b/tests/test_export_control.py @@ -0,0 +1,263 @@ +"""Pause / resume an in-progress export, and kill-on-delete. + +The single export worker tracks the ffmpeg child of the running job so the +HTTP layer can pause it (SIGSTOP), resume it (SIGCONT), or kill it when the +job is deleted mid-render. A killed job unwinds via _ExportCancelled without +being marked failed (its row is being deleted). +""" +from __future__ import annotations + +import signal +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.services import exporter +from web.services.exporter import ExportWorker, reconcile_orphan_jobs + + +async def _noop(_event): # broadcast stub + pass + + +@pytest.fixture +def db(tmp_path): + return Database(str(tmp_path / "t.db")) + + +def _job(db: Database, jid: int, state: str = "running") -> None: + with db.write() as c: + c.execute( + "INSERT INTO export_jobs (id, type, clip_ids, state, created_at) " + "VALUES (?, 'timeline', '{}', ?, 0)", + (jid, state), + ) + + +def _state(db: Database, jid: int) -> str: + with db.conn() as c: + return c.execute( + "SELECT state FROM export_jobs WHERE id=?", (jid,) + ).fetchone()["state"] + + +class _FakeProc: + def __init__(self): + self.signals: list = [] + self.killed = False + + def send_signal(self, sig): + self.signals.append(sig) + + def kill(self): + self.killed = True + + +def _worker(db: Database) -> ExportWorker: + return ExportWorker(db=db, provider=MagicMock(), broadcast=_noop) + + +# --- pause / resume --- + +async def test_pause_signals_stop_and_sets_state(db): + w = _worker(db) + _job(db, 7, "running") + w._current_job_id = 7 + w._current_proc = _FakeProc() + assert await w.pause(7) is True + assert signal.SIGSTOP in w._current_proc.signals + assert w._paused is True + assert _state(db, 7) == "paused" + + +async def test_resume_signals_cont_and_sets_state(db): + w = _worker(db) + _job(db, 7, "paused") + fake = _FakeProc() + w._current_job_id = 7 + w._current_proc = fake + w._paused = True + assert await w.resume(7) is True + assert signal.SIGCONT in fake.signals + assert w._paused is False + assert _state(db, 7) == "running" + + +async def test_pause_false_for_non_current_job(db): + w = _worker(db) + _job(db, 7, "running") + w._current_job_id = 7 + w._current_proc = _FakeProc() + assert await w.pause(99) is False + assert _state(db, 7) == "running" + + +# --- cancel / kill on delete --- + +async def test_cancel_kills_current_proc(db): + w = _worker(db) + fake = _FakeProc() + w._current_job_id = 7 + w._current_proc = fake + assert await w.cancel(7) is True + assert fake.killed is True + assert w._cancel_current is True + + +async def test_cancel_false_when_job_not_running(db): + w = _worker(db) + assert await w.cancel(7) is False + + +async def test_run_ffmpeg_raises_when_cancelled(db): + w = _worker(db) + w._current_job_id = 7 + w._cancel_current = True + with pytest.raises(exporter._ExportCancelled): + await w._run_ffmpeg(7, ["-y", "out.mp4"], 1.0) + + +async def test_run_ffmpeg_silences_libva_info_chatter(db, monkeypatch): + """ffmpeg runs with LIBVA_MESSAGING_LEVEL=1 so the QSV/VAAPI driver only + logs real errors, not its benign init handshake — without dropping the + rest of the inherited environment.""" + w = _worker(db) + captured: dict = {} + + class _Stream: + async def readline(self): + return b"" # immediate EOF -> pump loops exit at once + + class _Proc: + def __init__(self): + self.stdout = _Stream() + self.stderr = _Stream() + self.returncode = 0 + + async def wait(self): + return 0 + + async def fake_exec(*args, **kwargs): + captured["env"] = kwargs.get("env") + return _Proc() + + monkeypatch.setattr(exporter.asyncio, "create_subprocess_exec", fake_exec) + rc, err = await w._run_ffmpeg(7, ["-y", "out.mp4"], 1.0) + + assert rc == 0 + assert captured["env"]["LIBVA_MESSAGING_LEVEL"] == "1" + assert "PATH" in captured["env"] # parent env preserved + + +# --- _process: cancelled vs real failure --- + +async def test_process_discards_cancelled_job_without_failing(db, monkeypatch): + w = _worker(db) + _job(db, 7, "running") + + async def cancelled(_job): + raise exporter._ExportCancelled + + monkeypatch.setattr(w, "_run_job", cancelled) + await w._process({"id": 7}) + # row is being deleted by the endpoint; must NOT be flipped to failed + assert _state(db, 7) == "running" + + +async def test_process_marks_real_error_failed(db, monkeypatch): + w = _worker(db) + _job(db, 8, "running") + + async def boom(_job): + raise ValueError("nope") + + monkeypatch.setattr(w, "_run_job", boom) + await w._process({"id": 8}) + assert _state(db, 8) == "failed" + + +# --- restart reconcile includes paused --- + +def test_reconcile_marks_paused_and_running_failed(db): + _job(db, 1, "paused") + _job(db, 2, "running") + _job(db, 3, "done") + n = reconcile_orphan_jobs(db) + assert n == 2 + assert _state(db, 1) == "failed" + assert _state(db, 2) == "failed" + assert _state(db, 3) == "done" + + +# --- event-loop responsiveness --- + +async def test_pop_next_does_not_block_loop_while_db_busy(db): + """_pop_next runs a write transaction; with the DB lock held by a + worker thread it must wait off the loop, not freeze the server.""" + import asyncio + import threading + + w = _worker(db) + _job(db, 3, "queued") + + held = threading.Event() + release = threading.Event() + + def _hold_lock(): + with db.write(): + held.set() + release.wait(timeout=5.0) + + t = threading.Thread(target=_hold_lock, daemon=True) + t.start() + assert held.wait(timeout=5.0) + + ticks = 0 + + async def _ticker(): + nonlocal ticks + while True: + await asyncio.sleep(0.02) + ticks += 1 + + async def _call_pop(): + res = w._pop_next() + if asyncio.iscoroutine(res): + res = await res + return res + + tick_task = asyncio.create_task(_ticker()) + pop_task = asyncio.create_task(_call_pop()) + try: + await asyncio.sleep(0.3) + release.set() + job = await asyncio.wait_for(pop_task, timeout=5.0) + finally: + tick_task.cancel() + release.set() + t.join(timeout=5.0) + + assert job is not None and job["id"] == 3 + assert ticks >= 5, f"event loop starved while DB was busy ({ticks} ticks)" + + +# --- shutdown --- + +async def test_stop_unfreezes_and_kills_inflight_child(db): + """A paused job's encoder is SIGSTOP'd; shutdown must SIGCONT it + before killing or the frozen ffmpeg outlives the server. A plain + running child must be killed too — stop() used to abandon it.""" + w = _worker(db) + _job(db, 9, "paused") + w._current_job_id = 9 + w._paused = True + w._resume.clear() + proc = _FakeProc() + w._current_proc = proc + + await w.stop() + + assert signal.SIGCONT in proc.signals, "paused child never resumed" + assert proc.killed, "in-flight child not killed on shutdown" + assert w._resume.is_set(), "paused job left parked forever" diff --git a/tests/test_export_filmstrip_endpoint.py b/tests/test_export_filmstrip_endpoint.py new file mode 100644 index 0000000..f46d368 --- /dev/null +++ b/tests/test_export_filmstrip_endpoint.py @@ -0,0 +1,137 @@ +"""Tests for GET /api/exports/{job_id}/filmstrip.jpg and DELETE cache cleanup.""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + + +class _FakeMqttService: + def __init__(self, **kwargs): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): + return {"state": "idle", "detail": None, "last_published_at": None} + + +@pytest.fixture +def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) + + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + yield c + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def _insert_job(client, state, output_path): + app = client.app + with app.state.db.write() as c: + cur = c.execute( + "INSERT INTO export_jobs (type, clip_ids, state, created_at, " + "output_path) VALUES ('join_front', '[1]', ?, 0, ?)", + (state, output_path), + ) + return cur.lastrowid + + +def test_list_jobs_reports_has_preview(logged_in_client, tmp_path): + """The jobs list flags whether each done job's filmstrip is cached yet, so + the UI can show a 'generating' placeholder until the strip exists.""" + from web.services import export_preview + app = logged_in_client.app + recordings = app.state.settings_provider.get().recordings + done_with = _insert_job(logged_in_client, "done", str(tmp_path / "a.mp4")) + done_without = _insert_job(logged_in_client, "done", str(tmp_path / "b.mp4")) + running = _insert_job(logged_in_client, "running", None) + _pv = Path(export_preview.preview_path(recordings, done_with)) + _pv.parent.mkdir(parents=True, exist_ok=True) + _pv.write_bytes( + b"\xff\xd8\xff\xd9" + ) + + jobs = {j["id"]: j for j in logged_in_client.get("/api/exports").json()["jobs"]} + assert jobs[done_with]["has_preview"] is True + assert jobs[done_without]["has_preview"] is False + assert jobs[running]["has_preview"] is False + + +def test_filmstrip_jpg_streams_cached_sprite(logged_in_client, tmp_path): + from pathlib import Path + + from web.services import export_preview + app = logged_in_client.app + recordings = app.state.settings_provider.get().recordings + jid = _insert_job(logged_in_client, "done", str(tmp_path / "out.mp4")) + _pv = Path(export_preview.preview_path(recordings, jid)) + _pv.parent.mkdir(parents=True, exist_ok=True) + _pv.write_bytes(b"\xff\xd8\xff\xd9") + r = logged_in_client.get(f"/api/exports/{jid}/filmstrip.jpg") + assert r.status_code == 200 + assert r.headers["content-type"] == "image/jpeg" + assert "max-age" in r.headers.get("cache-control", "") + + +def test_filmstrip_jpg_never_generates_at_request_time(logged_in_client, tmp_path, monkeypatch): + from web.services import export_preview + + async def _boom(*a, **k): + raise AssertionError("endpoint must not generate previews") + + monkeypatch.setattr(export_preview, "ensure_export_preview", _boom) + jid = _insert_job(logged_in_client, "done", str(tmp_path / "out.mp4")) + # No cached sprite present -> must return placeholder WITHOUT calling ensure_*. + r = logged_in_client.get(f"/api/exports/{jid}/filmstrip.jpg") + assert r.status_code == 200 + assert r.headers["content-type"] == "image/png" + + +def test_filmstrip_jpg_placeholder_for_running_job(logged_in_client, tmp_path): + jid = _insert_job(logged_in_client, "running", None) + r = logged_in_client.get(f"/api/exports/{jid}/filmstrip.jpg") + assert r.status_code == 200 + assert r.headers["content-type"] == "image/png" # 1x1 placeholder + + +def test_filmstrip_jpg_placeholder_for_unknown_job(logged_in_client): + r = logged_in_client.get("/api/exports/999999/filmstrip.jpg") + assert r.status_code == 200 + assert r.headers["content-type"] == "image/png" + + +def test_delete_removes_cached_preview(logged_in_client, tmp_path, monkeypatch): + from web.services import export_preview + app = logged_in_client.app + recordings = app.state.settings_provider.get().recordings + jid = _insert_job(logged_in_client, "done", None) + pv = export_preview.preview_path(recordings, jid) + Path(pv).parent.mkdir(parents=True, exist_ok=True) + Path(pv).write_bytes(b"\xff\xd8\xff\xd9") + assert os.path.exists(pv) + csrf = logged_in_client.get("/api/auth/csrf").json()["csrf"] + r = logged_in_client.delete( + f"/api/exports/{jid}", headers={"x-csrf-token": csrf} + ) + assert r.status_code == 200 + assert not os.path.exists(pv) diff --git a/tests/test_export_hwaccel.py b/tests/test_export_hwaccel.py new file mode 100644 index 0000000..5c8e87e --- /dev/null +++ b/tests/test_export_hwaccel.py @@ -0,0 +1,257 @@ +"""Hardware-encoder command construction for exports. + +VAAPI encoding needs the frames on the GPU: a global ``-vaapi_device`` plus +a ``format=nv12,hwupload`` tail on the filter chain. Without them ffmpeg +fails with ``Invalid argument`` the moment any filter (scale/setsar/PiP) +is in the chain — which is every export. videotoolbox/nvenc accept software +frames directly, so they get neither. +""" +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.routers.exports import _resolve_default_encoder +from web.services import exporter +from web.services.exporter import ExportWorker + +# --- helpers --- + +def _state(prefs_encoder, available): + snap = SimpleNamespace(export_encoder_pref=prefs_encoder) + return SimpleNamespace( + settings_provider=SimpleNamespace(get=lambda: snap), + export_encoders=available, + ) + + +def test_auto_prefers_qsv_over_vaapi(): + st = _state("auto", {"qsv": True, "vaapi": True, "software": True}) + assert _resolve_default_encoder(st) == "qsv" + + +def test_auto_falls_back_to_vaapi_when_no_qsv(): + st = _state("auto", {"qsv": False, "vaapi": True, "software": True}) + assert _resolve_default_encoder(st) == "vaapi" + + +def test_hw_init_args_only_for_vaapi(): + assert exporter._hw_init_args("vaapi") == ["-vaapi_device", "/dev/dri/renderD128"] + assert exporter._hw_init_args("software") == [] + assert exporter._hw_init_args("videotoolbox") == [] + assert exporter._hw_init_args("nvenc") == [] + + +def test_hw_init_args_qsv_creates_device(): + assert exporter._hw_init_args("qsv") == [ + "-init_hw_device", "qsv=hw", "-filter_hw_device", "hw", + ] + + +def test_hw_init_args_vaapi_unchanged(): + assert exporter._hw_init_args("vaapi") == [ + "-vaapi_device", "/dev/dri/renderD128", + ] + + +def test_hw_decode_args_only_for_qsv(): + assert exporter._hw_decode_args("qsv") == [ + "-hwaccel", "qsv", "-hwaccel_output_format", "qsv", + ] + assert exporter._hw_decode_args("vaapi") == [] + assert exporter._hw_decode_args("software") == [] + assert exporter._hw_decode_args("videotoolbox") == [] + + +def test_hw_upload_filter_only_for_vaapi(): + assert exporter._hw_upload_filter("vaapi") == "format=nv12,hwupload" + assert exporter._hw_upload_filter("software") == "" + assert exporter._hw_upload_filter("nvenc") == "" + + +# --- timeline export wiring (the reported failure) --- + +@pytest.fixture +def db(tmp_path: Path) -> Database: + return Database(str(tmp_path / "t.db")) + + +async def _noop(_event): # broadcast stub + pass + + +def _insert_clip(db: Database, path: str, ts: int, dur: float = 60.0) -> None: + with db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(path, basename, group_name, timestamp, camera, sequence, " + " event_type, has_gpx, gps_examined, scanned_at, duration_s) " + "VALUES (?, ?, '2026-01-01', ?, 'F', 1, 'normal', 0, 0, 0, ?)", + (path, path.split("/")[-1], ts, dur), + ) + + +async def _capture_timeline(db, tmp_path, monkeypatch, encoder): + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_noop) + base = 1_000_000 + _insert_clip(db, str(tmp_path / "f.mp4"), base, 60.0) + + captured: list[list[str]] = [] + + async def fake_ffmpeg(job_id, args, total, **kw): + captured.append(list(args)) + Path(args[-1]).write_bytes(b"\0") + return 0, "" + + async def fake_res(_path): + return (1920, 1080) + + monkeypatch.setattr(worker, "_run_ffmpeg", fake_ffmpeg) + monkeypatch.setattr(worker, "_probe_resolution", fake_res) + + segments = [{"channel": "front", "start_ts": base + 10, "end_ts": base + 30}] + await worker._run_timeline({"id": 1}, segments, encoder, str(tmp_path / "out.mp4")) + + # the per-segment encode is the call carrying the scale filter + return next(a for a in captured + if "-vf" in a and "scale" in a[a.index("-vf") + 1]) + + +async def test_timeline_vaapi_adds_device_and_hwupload(db, tmp_path, monkeypatch): + seg = await _capture_timeline(db, tmp_path, monkeypatch, "vaapi") + assert seg[seg.index("-vaapi_device") + 1] == "/dev/dri/renderD128" + assert seg.index("-vaapi_device") < seg.index("-i") # global, before input + assert seg[seg.index("-vf") + 1] == "scale=1920:1080,setsar=1,format=nv12,hwupload" + assert "h264_vaapi" in seg + + +async def test_timeline_software_has_no_hw_args(db, tmp_path, monkeypatch): + seg = await _capture_timeline(db, tmp_path, monkeypatch, "software") + assert "-vaapi_device" not in seg + assert seg[seg.index("-vf") + 1] == "scale=1920:1080,setsar=1" + assert "libx264" in seg + + +def test_video_codec_args_qsv_uses_icq(): + args = exporter.video_codec_args("qsv") + assert args == [ + "-c:v", "h264_qsv", "-global_quality", "23", "-look_ahead", "0", + ] + + +def test_video_codec_args_vaapi_unchanged(): + # Regression guard: VAAPI path must not drift. + assert exporter.video_codec_args("vaapi") == [ + "-c:v", "h264_vaapi", "-rc_mode", "CQP", "-qp", "24", + ] + + +def test_scale_filter_dialects(): + # software/vaapi keep the exact legacy string (regression guard) + assert exporter._scale_filter(1920, 1080, "software") == "scale=1920:1080,setsar=1" + assert exporter._scale_filter(1920, 1080, "vaapi") == "scale=1920:1080,setsar=1" + # qsv uses the VPP scaler and drops setsar (set by the encoder) + assert exporter._scale_filter(1920, 1080, "qsv") == "scale_qsv=w=1920:h=1080" + + +async def test_timeline_qsv_uses_gpu_chain(db, tmp_path, monkeypatch): + seg = await _capture_timeline(db, tmp_path, monkeypatch, "qsv") + # device init present, before input + assert seg[seg.index("-init_hw_device") + 1] == "qsv=hw" + # per-input decode flags present, before -i + assert "-hwaccel" in seg and seg[seg.index("-hwaccel") + 1] == "qsv" + assert seg.index("-hwaccel") < seg.index("-i") + # qsv scaler, NO setsar, NO hwupload + vf = seg[seg.index("-vf") + 1] + assert vf == "scale_qsv=w=1920:h=1080" + assert "setsar" not in vf and "hwupload" not in vf + assert "h264_qsv" in seg + + +def test_pip_filter_complex_software_unchanged(): + fc = exporter._pip_filter_complex("top_right", main="front") + assert fc == ( + "[1:v]scale=iw/4:ih/4[pip];" + "[0:v][pip]overlay=W-w-20:20" + ) + + +def test_pip_filter_complex_qsv_uses_vpp(): + fc = exporter._pip_filter_complex("top_right", main="front", encoder="qsv") + assert fc == ( + "[1:v]scale_qsv=w=iw/4:h=ih/4[pip];" + "[0:v][pip]overlay_qsv=x=W-w-20:y=20" + ) + + +def test_qsv_probe_command_exercises_mfx(monkeypatch): + captured = {} + + def fake_run(cmd, **kw): + captured["cmd"] = cmd + class R: # noqa: D401 - tiny stub + returncode = 0 + return R() + + monkeypatch.setattr(exporter.subprocess, "run", fake_run) + monkeypatch.setattr(exporter.shutil, "which", lambda _x: "/usr/local/bin/ffmpeg") + + assert exporter._test_encoder_sync("qsv") is True + cmd = captured["cmd"] + # device init + qsv filter + qsv encoder all present + assert "-init_hw_device" in cmd and "qsv=hw" in cmd + assert any("scale_qsv" in c for c in cmd) + assert "h264_qsv" in cmd + + +async def test_finish_ok_schedules_export_preview(db, monkeypatch): + import asyncio + + from web.services import export_preview + + calls = [] + + async def fake_ensure(recordings, job_id, output_path, duration_s): + calls.append((job_id, output_path)) + return None + + monkeypatch.setattr(export_preview, "ensure_export_preview", fake_ensure) + + # A job row to update. + with db.write() as c: + c.execute( + "INSERT INTO export_jobs (id, type, clip_ids, state, created_at) " + "VALUES (5, 'join_front', '[1]', 'running', 0)" + ) + + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_noop) + worker._finish(5, True, None, "/recordings/.exports/5.mp4") + await asyncio.sleep(0) # let the scheduled task run + assert calls == [(5, "/recordings/.exports/5.mp4")] + + +async def test_finish_failure_does_not_schedule_preview(db, monkeypatch): + import asyncio + + from web.services import export_preview + + calls = [] + + async def fake_ensure(recordings, job_id, output_path, duration_s): + calls.append(job_id) + return None + + monkeypatch.setattr(export_preview, "ensure_export_preview", fake_ensure) + with db.write() as c: + c.execute( + "INSERT INTO export_jobs (id, type, clip_ids, state, created_at) " + "VALUES (6, 'join_front', '[1]', 'running', 0)" + ) + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_noop) + worker._finish(6, False, "boom", None) + await asyncio.sleep(0) + assert calls == [] diff --git a/tests/test_export_partial_cleanup.py b/tests/test_export_partial_cleanup.py new file mode 100644 index 0000000..445e956 --- /dev/null +++ b/tests/test_export_partial_cleanup.py @@ -0,0 +1,123 @@ +"""Failed/cancelled exports must not leave orphaned files in .exports. + +ffmpeg wrote straight to the final {job_id}.mp4, so a cancel or +failure left a partial that was unreferenced (output_path NULL) yet +counted against the recordings quota. Outputs now stage to a .part +name renamed only on verified success; failures/cancels remove the +partial, and a startup sweep clears pre-existing orphans. +""" +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.services import exporter as exp_mod +from web.services.exporter import ExportWorker, _ExportCancelled + + +async def _noop(_e): + pass + + +@pytest.fixture +def env(tmp_path: Path): + rec = tmp_path / "rec" + (rec / exp_mod.EXPORT_DIR_NAME).mkdir(parents=True) + db = Database(str(rec / "t.db")) + provider = MagicMock() + provider.get.return_value = MagicMock(recordings=str(rec)) + worker = ExportWorker(db=db, provider=provider, broadcast=_noop) + return rec, db, worker + + +def _job(db: Database, jid: int, state: str = "running") -> None: + with db.write() as c: + c.execute( + "INSERT INTO export_jobs (id, type, clip_ids, state, " + "progress, created_at) VALUES (?, 'pip', '{}', ?, 0, 0)", + (jid, state), + ) + + +def test_partial_path_keeps_inferable_extension(tmp_path: Path): + """ffmpeg chooses its muxer from the output filename's extension, so + the staged partial must end in a real container extension (.mp4), not + a bare .part it can't infer (regression: '{id}.mp4.part' made ffmpeg + fail with 'Unable to choose an output format').""" + rec = str(tmp_path) + part = exp_mod._partial_path(rec, 3) + assert part.endswith(".mp4"), part + assert ".part" in Path(part).name + assert part != exp_mod._output_path(rec, 3) + + +async def test_finish_success_renames_part_to_final(env): + rec, db, worker = env + _job(db, 5) + part = exp_mod._partial_path(str(rec), 5) + Path(part).write_bytes(b"video-bytes") + + worker._finish(5, True, None, part) + + final = exp_mod._output_path(str(rec), 5) + assert Path(final).read_bytes() == b"video-bytes" + assert not Path(part).exists() + with db.conn() as c: + assert c.execute( + "SELECT output_path FROM export_jobs WHERE id=5" + ).fetchone()["output_path"] == final + + +async def test_finish_failure_removes_partial(env): + rec, db, worker = env + _job(db, 6) + part = exp_mod._partial_path(str(rec), 6) + Path(part).write_bytes(b"half") + + worker._finish(6, False, "ffmpeg exit 1", None) + + assert not Path(part).exists(), "failed export left a partial behind" + + +async def test_cancel_removes_partial(env): + rec, db, worker = env + _job(db, 7) + part = exp_mod._partial_path(str(rec), 7) + Path(part).write_bytes(b"interrupted") + + async def _boom(job): + worker._current_job_id = job["id"] + raise _ExportCancelled() + + worker._run_job = _boom + await worker._process({"id": 7}) + + assert not Path(part).exists(), "cancelled export left a partial behind" + + +def test_startup_sweep_removes_orphans(env): + rec, db, worker = env + edir = rec / exp_mod.EXPORT_DIR_NAME + # Orphaned partial from a crashed render. + (edir / "9.mp4.part").write_bytes(b"x") + # Pre-fix orphan: a {id}.mp4 with no done row. + (edir / "10.mp4").write_bytes(b"y") + # A legitimately finished export must survive. + (edir / "11.mp4").write_bytes(b"keep") + with db.write() as c: + c.execute( + "INSERT INTO export_jobs (id, type, clip_ids, state, progress, " + "created_at, output_path) VALUES " + "(11, 'pip', '{}', 'done', 1.0, 0, ?)", + (str(edir / "11.mp4"),), + ) + + removed = exp_mod.sweep_orphan_exports(db, str(rec)) + + assert not (edir / "9.mp4.part").exists() + assert not (edir / "10.mp4").exists() + assert (edir / "11.mp4").exists() + assert removed == 2 diff --git a/tests/test_export_preview.py b/tests/test_export_preview.py new file mode 100644 index 0000000..2a26810 --- /dev/null +++ b/tests/test_export_preview.py @@ -0,0 +1,83 @@ +"""Tests for the export-job filmstrip preview service (ffmpeg mocked).""" +from __future__ import annotations + +import os +from pathlib import Path + +from web.services import export_preview + + +def test_preview_path_under_cache_dir(tmp_path: Path): + sp = export_preview.preview_path(str(tmp_path), 42) + assert sp.endswith(os.path.join(".export_previews", "42.jpg")) + # Pure computation: the directory is created at generation time + # (see test_preview_path_computation_has_no_side_effects). + + +def test_preview_timestamps_even_midpoints(): + ts = export_preview.preview_timestamps(100.0, n=10) + assert ts == [5.0, 15.0, 25.0, 35.0, 45.0, 55.0, 65.0, 75.0, 85.0, 95.0] + + +def test_preview_timestamps_degrades_for_unknown_duration(): + assert export_preview.preview_timestamps(0.0) == [0.0] + assert export_preview.preview_timestamps(None) == [0.0] + + +async def test_ensure_none_when_ffmpeg_missing(tmp_path: Path, monkeypatch): + monkeypatch.setattr(export_preview.shutil, "which", lambda _n: None) + out = tmp_path / "out.mp4" + out.write_bytes(b"\x00") + assert await export_preview.ensure_export_preview( + str(tmp_path), 1, str(out), 60.0) is None + + +async def test_ensure_none_when_output_missing(tmp_path: Path, monkeypatch): + monkeypatch.setattr(export_preview.shutil, "which", lambda _n: "/usr/bin/ffmpeg") + assert await export_preview.ensure_export_preview( + str(tmp_path), 1, str(tmp_path / "nope.mp4"), 60.0) is None + + +async def test_ensure_cache_hit_skips_generation(tmp_path: Path, monkeypatch): + sp = export_preview.preview_path(str(tmp_path), 7) + os.makedirs(os.path.dirname(sp), exist_ok=True) + Path(sp).write_bytes(b"\xff\xd8\xff\xd9") # pre-seeded sprite + + async def _boom(*a, **k): + raise AssertionError("should not generate on cache hit") + + monkeypatch.setattr(export_preview.filmstrip, "generate_sprite_at", _boom) + got = await export_preview.ensure_export_preview( + str(tmp_path), 7, str(tmp_path / "out.mp4"), 60.0) + assert got == sp + + +async def test_ensure_generates_and_returns_path(tmp_path: Path, monkeypatch): + monkeypatch.setattr(export_preview.shutil, "which", lambda _n: "/usr/bin/ffmpeg") + out = tmp_path / "out.mp4" + out.write_bytes(b"\x00") + + async def fake_gen(ffmpeg, video_path, sprite, timestamps): + Path(sprite).write_bytes(b"\xff\xd8\xff\xd9") + assert len(timestamps) == export_preview.N_FRAMES + return True + + monkeypatch.setattr(export_preview.filmstrip, "generate_sprite_at", fake_gen) + got = await export_preview.ensure_export_preview( + str(tmp_path), 9, str(out), 100.0) + assert got == export_preview.preview_path(str(tmp_path), 9) + assert os.path.getsize(got) > 0 + + +def test_preview_path_computation_has_no_side_effects(tmp_path): + """Computing a cache path must not create directories. The old + makedirs side effect, combined with tests passing bare MagicMock + providers (whose fspath serialises to a relative path), littered + the repo root with MagicMock/ directories — and meant a mere + GET /api/exports listing wrote to disk.""" + from web.services import export_preview + + rec = tmp_path / "rec" # deliberately never created + p = export_preview.preview_path(str(rec), 5) + assert p.endswith("5.jpg") + assert not rec.exists(), "preview_path created directories" diff --git a/tests/test_exporter_finish.py b/tests/test_exporter_finish.py index 343ea88..3f10ade 100644 --- a/tests/test_exporter_finish.py +++ b/tests/test_exporter_finish.py @@ -70,6 +70,96 @@ async def test_finish_failed_job_does_not_violate_progress_constraint( assert row["finished_at"] is not None +async def test_finish_snapshots_output_size_and_duration( + db: Database, tmp_path, monkeypatch, +) -> None: + """A successful job records the output's byte size and length on the row so + the export list can show them without re-probing.""" + from web.services import durations + + out = tmp_path / "out.mp4" + out.write_bytes(b"\0" * 4096) + monkeypatch.setattr(durations, "_probe_duration_mvhd", lambda p: 73.5) + + job_id = _insert_running_job(db, progress=0.9) + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_async_noop) + worker._finish(job_id, ok=True, err=None, output_path=str(out)) + + with db.conn() as c: + row = c.execute( + "SELECT * FROM export_jobs WHERE id=?", (job_id,) + ).fetchone() + assert row["output_size"] == 4096 + assert row["output_duration_s"] == pytest.approx(73.5) + + +async def test_finish_failure_leaves_output_stats_null( + db: Database, +) -> None: + """A failed job has no usable output, so size/length stay NULL.""" + job_id = _insert_running_job(db, progress=0.3) + worker = ExportWorker(db=db, provider=MagicMock(), broadcast=_async_noop) + worker._finish(job_id, ok=False, err="boom", output_path=None) + + with db.conn() as c: + row = c.execute( + "SELECT * FROM export_jobs WHERE id=?", (job_id,) + ).fetchone() + assert row["output_size"] is None + assert row["output_duration_s"] is None + + +async def test_make_export_preview_broadcasts_ready_on_success( + db: Database, monkeypatch, +) -> None: + """When the filmstrip lands, the worker emits export_preview_ready so the + UI can swap its 'generating' placeholder for the real strip.""" + from web.services import export_preview + + events = [] + + async def capture(ev): + events.append(ev) + + snap = MagicMock() + snap.recordings = "/rec" + provider = MagicMock() + provider.get.return_value = snap + worker = ExportWorker(db=db, provider=provider, broadcast=capture) + + async def fake_ensure(recordings, job_id, output_path, duration_s): + return "/rec/.export_previews/9.jpg" + + monkeypatch.setattr(export_preview, "ensure_export_preview", fake_ensure) + await worker._make_export_preview(9, "/tmp/out.mp4") + assert {"type": "export_preview_ready", "job_id": 9} in events + + +async def test_make_export_preview_silent_when_generation_fails( + db: Database, monkeypatch, +) -> None: + """No file -> no ready event (the placeholder simply stays).""" + from web.services import export_preview + + events = [] + + async def capture(ev): + events.append(ev) + + snap = MagicMock() + snap.recordings = "/rec" + provider = MagicMock() + provider.get.return_value = snap + worker = ExportWorker(db=db, provider=provider, broadcast=capture) + + async def fake_ensure(*a, **k): + return None + + monkeypatch.setattr(export_preview, "ensure_export_preview", fake_ensure) + await worker._make_export_preview(9, "/tmp/out.mp4") + assert events == [] + + async def test_finish_done_job_writes_full_progress( db: Database, ) -> None: diff --git a/tests/test_filmstrip.py b/tests/test_filmstrip.py new file mode 100644 index 0000000..0e18ea3 --- /dev/null +++ b/tests/test_filmstrip.py @@ -0,0 +1,312 @@ +"""Tests for the filmstrip sprite service (ffmpeg mocked).""" +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path + +from web.services import filmstrip, retention + + +def test_frame_count_basics(): + # 60s clip, one frame / 8s -> ceil(60/8) = 8 + assert filmstrip.frame_count(60.0) == 8 + # exact multiple + assert filmstrip.frame_count(16.0) == 2 + # short / zero / None always yields at least one tile + assert filmstrip.frame_count(3.0) == 1 + assert filmstrip.frame_count(0.0) == 1 + assert filmstrip.frame_count(None) == 1 + + +def test_paths_are_under_filmstrips_dir(tmp_path: Path): + rec = str(tmp_path) + sp = filmstrip.sprite_path(rec, 42) + mp = filmstrip.meta_path(rec, 42) + assert sp.endswith(os.path.join(".filmstrips", "42.jpg")) + assert mp.endswith(os.path.join(".filmstrips", "42.json")) + # accessing a path helper creates the cache dir + assert os.path.isdir(os.path.join(rec, ".filmstrips")) + + +async def test_ensure_returns_none_when_ffmpeg_missing(tmp_path: Path, monkeypatch): + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: None) + meta = await filmstrip.ensure_filmstrip( + str(tmp_path), 7, str(tmp_path / "clip.mp4"), 60.0 + ) + assert meta is None + + +async def test_ensure_cache_hit_reads_sidecar_without_ffmpeg(tmp_path: Path, monkeypatch): + rec = str(tmp_path) + # Pre-seed a cached sprite + sidecar. + Path(filmstrip.sprite_path(rec, 9)).write_bytes(b"\xff\xd8\xff\xd9") # tiny JPEG-ish + Path(filmstrip.meta_path(rec, 9)).write_text(json.dumps({ + "frames": 8, "interval_s": 8, "tile_w": 160, "tile_h": 90, "duration_s": 60.0, + })) + + # If ffmpeg were invoked this would explode — proves the cache short-circuits. + def _boom(*a, **k): + raise AssertionError("ffmpeg must not run on a cache hit") + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", _boom) + + meta = await filmstrip.ensure_filmstrip(rec, 9, str(tmp_path / "clip.mp4"), 60.0) + assert meta == filmstrip.FilmstripMeta(8, 8, 160, 90, 60.0) + + +async def test_ensure_generates_sprite_and_sidecar(tmp_path: Path, monkeypatch): + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: "/usr/bin/ffmpeg") + calls = _capture_all_exec(monkeypatch) + + meta = await filmstrip.ensure_filmstrip(rec, 5, "/rec/clip.mp4", 60.0) + + # Returned + persisted metadata (unchanged contract) + assert meta == filmstrip.FilmstripMeta(8, 8, 160, 90, 60.0) + assert os.path.exists(filmstrip.sprite_path(rec, 5)) + with open(filmstrip.meta_path(rec, 5)) as f: + assert json.load(f)["frames"] == 8 + + # One seek-extract per tile (read only near each 8s mark, not the whole + # file), then a single stitch pass into the sprite. + extracts = [c for c in calls if "-ss" in c] + tiles = [c for c in calls if any("tile=" in a for a in c)] + assert len(extracts) == 8 + assert [c[c.index("-ss") + 1] for c in extracts] == \ + ["0", "8", "16", "24", "32", "40", "48", "56"] + for c in extracts: + assert c[c.index("-i") + 1] == "/rec/clip.mp4" + assert "-frames:v" in c + assert c[c.index("-vf") + 1] == "scale=160:90" + assert "-an" in c + assert "-hwaccel" not in c # software only — hwaccel is slower here + assert len(tiles) == 1 + assert tiles[0][tiles[0].index("-vf") + 1] == "tile=8x1" + # Montage targets a staged temp name; the verified result is + # renamed onto the cache path (partial output must never land + # at the final path — see test_thumb_atomic.py). + assert tiles[0][-1] == filmstrip.sprite_path(rec, 5) + ".part.jpg" + + +def _capture_all_exec(monkeypatch): + """Patch create_subprocess_exec to record every call's argv and write a + stub output (each ffmpeg writes its last positional arg).""" + calls: list[list[str]] = [] + + class _Proc: + returncode = 0 + async def wait(self): + return 0 + + async def fake_exec(*args, **kwargs): + calls.append(list(args)) + Path(args[-1]).write_bytes(b"\xff\xd8\xff\xd9") + return _Proc() + + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", fake_exec) + return calls + + +async def test_ensure_short_clip_is_single_seek(tmp_path: Path, monkeypatch): + """A sub-INTERVAL clip yields one tile: a single seek at t=0.""" + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: "/usr/bin/ffmpeg") + calls = _capture_all_exec(monkeypatch) + + meta = await filmstrip.ensure_filmstrip(rec, 7, "/rec/c.mp4", 3.0) + assert meta.frames == 1 + extracts = [c for c in calls if "-ss" in c] + assert len(extracts) == 1 + assert extracts[0][extracts[0].index("-ss") + 1] == "0" + + +async def test_ensure_returns_none_on_ffmpeg_nonzero(tmp_path: Path, monkeypatch): + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: "/usr/bin/ffmpeg") + + class _FailProc: + returncode = 1 + async def wait(self): + return 1 + + async def fake_exec(*args, **kwargs): + return _FailProc() # writes nothing + + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", fake_exec) + meta = await filmstrip.ensure_filmstrip(rec, 6, "/rec/clip.mp4", 60.0) + assert meta is None + + +def test_retention_removes_filmstrip_sprite_and_sidecar(tmp_path: Path): + rec = str(tmp_path) + clip_file = tmp_path / "clip.mp4" + clip_file.write_bytes(b"\0") + + sp = filmstrip.sprite_path(rec, 11) + mp = filmstrip.meta_path(rec, 11) + Path(sp).write_bytes(b"\xff\xd8\xff\xd9") + Path(mp).write_text("{}") + + rec_row = {"id": 11, "path": str(clip_file)} + retention._delete_clip_files(rec_row, rec) + + assert not clip_file.exists() + assert not os.path.exists(sp) + assert not os.path.exists(mp) + + +class _HangProc: + """Fake ffmpeg child: kill() records, wait() counts body runs.""" + returncode = None + + def __init__(self): + self.killed = False + self.reaped = 0 + + def kill(self): + self.killed = True + + async def wait(self): + self.reaped += 1 + return 0 + + +async def _raise_timeout(coro, timeout): + # Close the inner proc.wait() coroutine so it isn't left un-awaited + # (the suite runs under filterwarnings=error), then simulate a timeout. + coro.close() + raise TimeoutError + + +async def test_ensure_reaps_child_on_timeout(tmp_path: Path, monkeypatch): + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _n: "/usr/bin/ffmpeg") + fake = _HangProc() + + async def fake_exec(*a, **k): + return fake + + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(filmstrip.asyncio, "wait_for", _raise_timeout) + + result = await filmstrip.ensure_filmstrip(rec, 99, "/x.mp4", 60.0) + assert result is None + assert fake.killed is True + assert fake.reaped == 1 # proc.wait() awaited after kill -> child reaped + + +# --- logging: the timeline feature must be debuggable via the Logs tab. +# The app log persists INFO+ from the ``viofosync.*` namespace, so these +# assert the filmstrip service emits on that logger so a NAS CPU spike is +# traceable to the exact clips being rendered. + + +async def test_generation_logs_start_and_done(tmp_path: Path, monkeypatch, caplog): + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: "/usr/bin/ffmpeg") + + class _FakeProc: + returncode = 0 + async def wait(self): + return 0 + + async def fake_exec(*args, **kwargs): + Path(args[-1]).write_bytes(b"\xff\xd8\xff\xd9") + return _FakeProc() + + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", fake_exec) + + with caplog.at_level(logging.INFO, logger="viofosync.filmstrip"): + await filmstrip.ensure_filmstrip(rec, 5, "/rec/clip.mp4", 60.0) + + msgs = [r.getMessage() for r in caplog.records] + assert any("generating clip=5" in m and "frames=8" in m for m in msgs) + assert any("clip=5 done" in m for m in msgs) + + +async def test_timeout_logs_warning(tmp_path: Path, monkeypatch, caplog): + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _n: "/usr/bin/ffmpeg") + fake = _HangProc() + + async def fake_exec(*a, **k): + return fake + + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(filmstrip.asyncio, "wait_for", _raise_timeout) + + with caplog.at_level(logging.INFO, logger="viofosync.filmstrip"): + await filmstrip.ensure_filmstrip(rec, 99, "/x.mp4", 60.0) + + warnings = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] + assert any("clip=99 generation failed" in m for m in warnings) + + +async def test_ffmpeg_nonzero_logs_warning(tmp_path: Path, monkeypatch, caplog): + rec = str(tmp_path) + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: "/usr/bin/ffmpeg") + + class _FailProc: + returncode = 1 + async def wait(self): + return 1 + + async def fake_exec(*args, **kwargs): + return _FailProc() + + monkeypatch.setattr(filmstrip.asyncio, "create_subprocess_exec", fake_exec) + with caplog.at_level(logging.INFO, logger="viofosync.filmstrip"): + await filmstrip.ensure_filmstrip(rec, 6, "/rec/clip.mp4", 60.0) + + warnings = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] + assert any("clip=6 generation failed" in m for m in warnings) + + +async def test_missing_ffmpeg_warns_once(tmp_path: Path, monkeypatch, caplog): + monkeypatch.setattr(filmstrip.shutil, "which", lambda _name: None) + monkeypatch.setattr(filmstrip, "_warned_no_ffmpeg", False) + + with caplog.at_level(logging.INFO, logger="viofosync.filmstrip"): + await filmstrip.ensure_filmstrip(str(tmp_path), 1, "/a.mp4", 60.0) + await filmstrip.ensure_filmstrip(str(tmp_path), 2, "/b.mp4", 60.0) + + no_ffmpeg = [ + r for r in caplog.records + if "ffmpeg not found" in r.getMessage() + ] + assert len(no_ffmpeg) == 1 # warned once, not once-per-clip + + +async def test_generate_sprite_at_extracts_given_timestamps(tmp_path, monkeypatch): + from pathlib import Path + seen = [] + + async def fake_run(cmd, timeout): + if "-ss" in cmd: + seen.append(cmd[cmd.index("-ss") + 1]) + Path(cmd[-1]).write_bytes(b"\xff\xd8\xff\xd9") # tiny JPEG-ish + return 0 + + monkeypatch.setattr(filmstrip, "_run_ffmpeg", fake_run) + ok = await filmstrip.generate_sprite_at( + "ffmpeg", "in.mp4", str(tmp_path / "s.jpg"), [1.5, 3.0, 4.5]) + assert ok is True + assert seen == ["1.5", "3.0", "4.5"] + + +async def test_generate_sprite_keeps_interval_timestamps(tmp_path, monkeypatch): + # Regression: the clip filmstrip must still seek at i*INTERVAL_S. + from pathlib import Path + seen = [] + + async def fake_run(cmd, timeout): + if "-ss" in cmd: + seen.append(cmd[cmd.index("-ss") + 1]) + Path(cmd[-1]).write_bytes(b"\xff\xd8\xff\xd9") + return 0 + + monkeypatch.setattr(filmstrip, "_run_ffmpeg", fake_run) + ok = await filmstrip._generate_sprite("ffmpeg", "in.mp4", str(tmp_path / "s.jpg"), 4) + assert ok is True + assert seen == ["0", "8", "16", "24"] # INTERVAL_S == 8 diff --git a/tests/test_filmstrip_endpoints.py b/tests/test_filmstrip_endpoints.py new file mode 100644 index 0000000..38225d7 --- /dev/null +++ b/tests/test_filmstrip_endpoints.py @@ -0,0 +1,149 @@ +"""Tests for GET /api/archive/clip/{id}/filmstrip[.jpg].""" +from __future__ import annotations + +from pathlib import Path + +import pytest + + +class _FakeMqttService: + def __init__(self, **kwargs): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): + return {"state": "idle", "detail": None, "last_published_at": None} + + +@pytest.fixture +def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) + + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + yield c, Path(str(tmp_recordings_dir)) + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def _insert_clip(app, clip_id: int, path: str, duration_s: float) -> None: + db = app.state.db + with db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(id, path, basename, group_name, timestamp, camera, " + " sequence, event_type, has_gpx, gps_examined, scanned_at, duration_s) " + "VALUES (?,?,?,?,?,?,?,?,0,0,?,?)", + (clip_id, path, f"{clip_id}.MP4", "2026-06-02", + 1_717_312_440, "F", clip_id, "normal", 1_717_312_440, duration_s), + ) + + +def test_filmstrip_meta_returns_slicing_info(logged_in_client, monkeypatch): + client, rec = logged_in_client + clip_file = rec / "clip.mp4" + clip_file.write_bytes(b"\0") + _insert_clip(client.app, 1, str(clip_file), 60.0) + + from web.services import filmstrip + + async def fake_ensure(recordings, clip_id, video_path, duration_s): + Path(filmstrip.sprite_path(recordings, clip_id)).write_bytes(b"\xff\xd8\xff\xd9") + return filmstrip.FilmstripMeta(8, 8, 160, 90, 60.0) + + monkeypatch.setattr("web.routers.archive.filmstrip.ensure_filmstrip", fake_ensure) + + r = client.get("/api/archive/clip/1/filmstrip") + assert r.status_code == 200 + body = r.json() + assert body["frames"] == 8 + assert body["interval_s"] == 8 + assert body["tile_w"] == 160 + assert body["sprite_url"] == "/api/archive/clip/1/filmstrip.jpg" + + +def test_filmstrip_meta_204_when_ffmpeg_unavailable(logged_in_client, monkeypatch): + client, rec = logged_in_client + clip_file = rec / "clip.mp4" + clip_file.write_bytes(b"\0") + _insert_clip(client.app, 2, str(clip_file), 60.0) + + async def fake_ensure(*a, **k): + return None + + monkeypatch.setattr("web.routers.archive.filmstrip.ensure_filmstrip", fake_ensure) + + r = client.get("/api/archive/clip/2/filmstrip") + assert r.status_code == 204 + + +def test_filmstrip_jpg_served(logged_in_client, monkeypatch): + client, rec = logged_in_client + clip_file = rec / "clip.mp4" + clip_file.write_bytes(b"\0") + _insert_clip(client.app, 3, str(clip_file), 60.0) + + from web.services import filmstrip + + async def fake_ensure(recordings, clip_id, video_path, duration_s): + Path(filmstrip.sprite_path(recordings, clip_id)).write_bytes(b"\xff\xd8\xff\xd9") + return filmstrip.FilmstripMeta(8, 8, 160, 90, 60.0) + + monkeypatch.setattr("web.routers.archive.filmstrip.ensure_filmstrip", fake_ensure) + + r = client.get("/api/archive/clip/3/filmstrip.jpg") + assert r.status_code == 200 + assert r.headers["content-type"].startswith("image/jpeg") + + +def test_filmstrip_jpg_404_when_ffmpeg_unavailable(logged_in_client, monkeypatch): + client, rec = logged_in_client + clip_file = rec / "clip.mp4" + clip_file.write_bytes(b"\0") + _insert_clip(client.app, 4, str(clip_file), 60.0) + + async def fake_ensure(*a, **k): + return None + + monkeypatch.setattr("web.routers.archive.filmstrip.ensure_filmstrip", fake_ensure) + + r = client.get("/api/archive/clip/4/filmstrip.jpg") + assert r.status_code == 404 + + +def test_filmstrip_jpg_404_when_sprite_missing(logged_in_client, monkeypatch): + # Defensive guard: meta is returned but the sprite file is absent + # (e.g. retention deleted it concurrently) -> 404, not a 500. + client, rec = logged_in_client + clip_file = rec / "clip.mp4" + clip_file.write_bytes(b"\0") + _insert_clip(client.app, 5, str(clip_file), 60.0) + + from web.services import filmstrip + + async def fake_ensure(recordings, clip_id, video_path, duration_s): + # Return valid meta but deliberately DO NOT write the sprite file. + return filmstrip.FilmstripMeta(8, 8, 160, 90, 60.0) + + monkeypatch.setattr("web.routers.archive.filmstrip.ensure_filmstrip", fake_ensure) + + r = client.get("/api/archive/clip/5/filmstrip.jpg") + assert r.status_code == 404 diff --git a/tests/test_gpx_corrupt.py b/tests/test_gpx_corrupt.py new file mode 100644 index 0000000..59f45a0 --- /dev/null +++ b/tests/test_gpx_corrupt.py @@ -0,0 +1,64 @@ +"""parse_moov must terminate on corrupt/truncated MP4 input. + +Power-loss recordings (exactly what dashcams produce) can contain a +zero-size or short child atom inside ``moov``; the walker must treat +any atom smaller than its own 8-byte header as corrupt and stop, +rather than looping forever re-reading the same offset. +""" +from __future__ import annotations + +import io +import struct +import threading + +from viofosync_lib import parse_moov + + +def _run_with_timeout(fh, timeout: float = 2.0): + """Run parse_moov in a thread so a regression fails fast instead + of hanging the suite.""" + result: list = [None] + done = threading.Event() + + def _target() -> None: + result[0] = parse_moov(fh) + done.set() + + t = threading.Thread(target=_target, daemon=True) + t.start() + finished = done.wait(timeout=timeout) + return finished, result[0] + + +def test_zero_size_child_atom_terminates(): + # moov atom (size 24) containing a child whose size field is 0 — + # the inner walk used to do `sub_offset += 0` forever. + data = ( + struct.pack(">I4s", 24, b"moov") + + struct.pack(">I4s", 0, b"free") + + b"\x00" * 8 + ) + finished, gps = _run_with_timeout(io.BytesIO(data)) + assert finished, "parse_moov hung on a zero-size child atom" + assert gps == [] + + +def test_undersized_child_atom_terminates(): + # Child size 4 < 8-byte header: corrupt; must not crawl/spin. + data = ( + struct.pack(">I4s", 24, b"moov") + + struct.pack(">I4s", 4, b"free") + + b"\x00" * 8 + ) + finished, gps = _run_with_timeout(io.BytesIO(data)) + assert finished, "parse_moov hung on an undersized child atom" + assert gps == [] + + +def test_truncated_file_mid_child_header_terminates(): + # File ends in the middle of a child header — get_atom_info's + # short read yields (0, '') and must end the walk. + data = struct.pack(">I4s", 64, b"moov") + b"\x00" * 3 + finished, gps = _run_with_timeout(io.BytesIO(data)) + assert finished, "parse_moov hung on a truncated child header" + assert gps == [] diff --git a/tests/test_hub.py b/tests/test_hub.py index 321b43d..e169620 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -303,3 +303,56 @@ def get(self_): assert any(e.get("type") == "dashcam_online" for e in ws.sent) # No sync_status follow-up. assert not any(e.get("type") == "sync_status" for e in ws.sent) + + +# ---- regression: one stalled client must not block the others ---- + +class _StalledWS: + """send_json applies indefinite backpressure (zombie tab, closed + laptop lid) — it never raises, it just never completes.""" + + def __init__(self): + self.accept_calls = 0 + + async def accept(self) -> None: + self.accept_calls += 1 + + async def send_json(self, _payload) -> None: + import asyncio + await asyncio.sleep(60) + + +class _HealthyWS: + def __init__(self): + self.received: list = [] + + async def accept(self) -> None: + pass + + async def send_json(self, payload) -> None: + self.received.append(payload) + + +async def test_stalled_client_does_not_block_broadcast(monkeypatch) -> None: + import asyncio + import time + + import web.services.hub as hub_mod + + monkeypatch.setattr(hub_mod, "SEND_TIMEOUT_S", 0.1, raising=False) + hub = Hub() + stalled = _StalledWS() + healthy = _HealthyWS() + hub._clients.update({stalled, healthy}) + + started = time.monotonic() + await asyncio.wait_for( + hub.broadcast({"type": "ping"}), timeout=2.0, + ) + elapsed = time.monotonic() - started + + assert {"type": "ping"} in healthy.received, \ + "healthy client starved by a stalled one" + assert elapsed < 1.5 + assert stalled not in hub._clients, "stalled client not evicted" + assert healthy in hub._clients diff --git a/tests/test_import_endpoints.py b/tests/test_import_endpoints.py index c30a46e..a70b3b3 100644 --- a/tests/test_import_endpoints.py +++ b/tests/test_import_endpoints.py @@ -85,7 +85,47 @@ def test_scan_lists_recognised_and_skipped(client): assert body["total_bytes"] == 10 assert [it["basename"] for it in body["recognised"]] == [ "2026_0101_080000_0001F.MP4"] - assert {s["name"] for s in body["skipped"]} == {"junk.bin"} + # Counts only — the endpoint must not leak skipped filenames. + assert body["skipped_count"] == 1 + assert "skipped" not in body + + +def test_present_reports_clips_already_in_archive(client): + c, rec = client + here = "2026_0101_080000_0001F.MP4" # complete copy -> present + partial = "2026_0102_090000_0002R.MP4" # archive smaller -> redo -> absent + gone = "2026_0103_100000_0003F.MP4" # not imported -> absent + (rec / "2026-01-01").mkdir() + (rec / "2026-01-01" / here).write_bytes(b"a" * 10) + (rec / "2026-01-02").mkdir() + (rec / "2026-01-02" / partial).write_bytes(b"a" * 3) + r = c.post("/api/import/present", json={"files": [ + {"name": here, "size": 10}, + {"name": partial, "size": 10}, + {"name": gone, "size": 10}, + ]}) + assert r.status_code == 200 + assert r.json()["present"] == [here] + + +def test_scan_marks_present_clips(client): + c, rec = client + card = rec / "import" / "DCIM" + card.mkdir(parents=True) + here = "2026_0101_080000_0001F.MP4" # already archived, full size + partial = "2026_0102_090000_0002R.MP4" # archived but truncated + fresh = "2026_0103_100000_0003F.MP4" # not in archive + (card / here).write_bytes(b"a" * 10) + (card / partial).write_bytes(b"b" * 10) + (card / fresh).write_bytes(b"c" * 10) + (rec / "2026-01-01").mkdir() + (rec / "2026-01-01" / here).write_bytes(b"a" * 10) + (rec / "2026-01-02").mkdir() + (rec / "2026-01-02" / partial).write_bytes(b"b" * 4) + r = c.post("/api/import/scan", json={}) + assert r.status_code == 200 + present = {it["basename"]: it["present"] for it in r.json()["recognised"]} + assert present == {here: True, partial: False, fresh: False} def test_scan_bad_path_400(client): diff --git a/tests/test_import_scan_no_leak.py b/tests/test_import_scan_no_leak.py new file mode 100644 index 0000000..1f2b8d3 --- /dev/null +++ b/tests/test_import_scan_no_leak.py @@ -0,0 +1,46 @@ +"""The import scan endpoint must not leak filenames of arbitrary +directories — the scan root can be any readable path the user types, +so returning every non-matching filename was an authenticated +directory-listing primitive. It reports counts only. +""" +from __future__ import annotations + +from types import SimpleNamespace + + +def test_scan_does_not_leak_skipped_filenames(monkeypatch): + from web.routers import imports as imports_router + + class _Item: + def __init__(self, basename, size): + self.basename = basename + self.size_bytes = size + + class _Manifest: + items = [_Item("2026_0101_080000_0001F.MP4", 10)] + total_bytes = 10 + skipped = [ + {"name": "id_rsa", "reason": "not_recognised"}, + {"name": "secret-budget.xlsx", "reason": "not_recognised"}, + ] + + monkeypatch.setattr(imports_router.importer, "scan_source", + lambda root: _Manifest()) + monkeypatch.setattr(imports_router.importer, "present_in_archive", + lambda snap, sizes: set()) + monkeypatch.setattr(imports_router.importer, "is_cross_volume", + lambda a, b: False) + monkeypatch.setattr(imports_router.importer, "scan_item_dict", + lambda it: {"basename": it.basename}) + monkeypatch.setattr(imports_router.os.path, "isdir", lambda p: True) + + snap = SimpleNamespace(import_path="/anything", recordings="/rec") + monkeypatch.setattr(imports_router, "_snap", lambda req: snap) + + body = imports_router.scan(request=None, body=imports_router._PathBody(path="/etc")) + + blob = str(body) + assert "id_rsa" not in blob and "secret-budget" not in blob, \ + "scan leaked arbitrary filenames" + assert body["skipped_count"] == 2 + assert "skipped" not in body diff --git a/tests/test_importer.py b/tests/test_importer.py index 522431c..9c4802d 100644 --- a/tests/test_importer.py +++ b/tests/test_importer.py @@ -84,6 +84,33 @@ def test_ingest_clip_places_file_and_records_origin(tmp_path: Path): assert rows[name]["event_type"] == "ro" +def test_ingest_clip_marks_preexisting_queue_row_done(tmp_path: Path): + # The dashcam listed this clip first (queued pending); the user then + # bulk web-uploads it. The import must flip the row to done, or the + # next Wi-Fi cycle re-tries the (now 404) download. + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + name = "2026_0519_074752_022262PF.MP4" + with db.write() as c: + c.execute( + "INSERT INTO download_queue (filename, source_dir, state, " + "enqueued_at) VALUES (?,?,?,0)", + (name, "/DCIM/Movie", "pending"), + ) + src = tmp_path / "usb" + src.mkdir() + (src / name).write_bytes(b"a" * 10) + man = importer.scan_source(str(src)) + res = importer.ingest_clip(db, _snap(rec), man.items[0], cross_volume=True) + assert res.status == "imported" + rows = _origin_rows(db) + assert rows[name]["state"] == "done" + assert rows[name]["manual"] == 1 + + def test_ingest_clip_cross_volume_copies_and_keeps_source(tmp_path: Path): from web.db import Database from web.services import importer @@ -104,7 +131,7 @@ def test_ingest_clip_cross_volume_copies_and_keeps_source(tmp_path: Path): assert rows[name]["manual"] == 1 -def test_ingest_clip_skips_duplicate(tmp_path: Path): +def test_ingest_clip_skips_complete_duplicate(tmp_path: Path): from web.db import Database from web.services import importer rec = tmp_path / "rec" @@ -112,14 +139,53 @@ def test_ingest_clip_skips_duplicate(tmp_path: Path): db = Database(str(rec / ".viofosync.db")) name = "2026_0101_080000_0001F.MP4" (rec / "2026-01-01").mkdir() - (rec / "2026-01-01" / name).write_bytes(b"existing") + (rec / "2026-01-01" / name).write_bytes(b"x" * 10) # same size -> complete src = tmp_path / "usb" src.mkdir() (src / name).write_bytes(b"a" * 10) man = importer.scan_source(str(src)) res = importer.ingest_clip(db, _snap(rec), man.items[0], cross_volume=True) assert res.status == "already_present" - assert (rec / "2026-01-01" / name).read_bytes() == b"existing" + assert (rec / "2026-01-01" / name).read_bytes() == b"x" * 10 + + +def test_ingest_clip_redoes_partial_archive_file(tmp_path: Path): + from web.db import Database + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + name = "2026_0101_080000_0001F.MP4" + (rec / "2026-01-01").mkdir() + (rec / "2026-01-01" / name).write_bytes(b"ab") # 2 bytes — truncated + src = tmp_path / "usb" + src.mkdir() + (src / name).write_bytes(b"a" * 10) + man = importer.scan_source(str(src)) + res = importer.ingest_clip(db, _snap(rec), man.items[0], cross_volume=True) + assert res.status == "imported" # partial gets redone + assert (rec / "2026-01-01" / name).read_bytes() == b"a" * 10 + + +def test_present_in_archive_matches_size_and_skips_partials(tmp_path: Path): + from web.services import importer + rec = tmp_path / "rec" + rec.mkdir() + full = "2026_0101_080000_0001F.MP4" # exact-size copy -> present + bigger = "2026_0102_090000_0002R.MP4" # archive larger -> keep, present + partial = "2026_0103_100000_0003F.MP4" # archive smaller -> redo, absent + gone = "2026_0104_110000_0004R.MP4" # not imported at all -> absent + (rec / "2026-01-01").mkdir() + (rec / "2026-01-01" / full).write_bytes(b"a" * 10) + (rec / "2026-01-02").mkdir() + (rec / "2026-01-02" / bigger).write_bytes(b"a" * 20) + (rec / "2026-01-03").mkdir() + (rec / "2026-01-03" / partial).write_bytes(b"a" * 3) + + present = importer.present_in_archive(_snap(rec), { + full: 10, bigger: 10, partial: 10, gone: 10, "notes.txt": 5, + }) + assert present == {full, bigger} def test_ingest_clip_restores_source_when_final_rename_fails(tmp_path, monkeypatch): diff --git a/tests/test_importer_staging.py b/tests/test_importer_staging.py new file mode 100644 index 0000000..e700e7c --- /dev/null +++ b/tests/test_importer_staging.py @@ -0,0 +1,133 @@ +"""Staging recovery: a crash must never cost the only copy of a clip. + +Same-volume folder ingest *moves* the source into ``.import_tmp`` +before renaming into the archive; the old ``_clean_staging`` deleted +everything there on the next run — destroying the only copy after a +crash between the two renames. Staged Viofo-named files are complete +by construction (atomic rename / post-verification rename), so they +are recovered. In-flight browser uploads stream to a ``.part`` name +so cleanup can tell them apart and never deletes a fresh one. +""" +from __future__ import annotations + +import os +import time +import types +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.services import importer + +NAME = "2026_0102_090000_0002F.MP4" + + +def _snap(rec: Path): + return types.SimpleNamespace( + recordings=str(rec), grouping="daily", gps_extract=False, + retention_disk_pct=0, recordings_quota_gb=0, + retention_protect_ro=True, retention_max_days=0, import_path="", + ) + + +@pytest.fixture +def env(tmp_path: Path): + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + staging = rec / importer.STAGING_DIRNAME + staging.mkdir() + return rec, db, staging + + +def test_recover_staging_reingests_complete_clip(env): + rec, db, staging = env + (staging / NAME).write_bytes(b"b" * 20) # crash left this here + + summary = importer.recover_staging(db, _snap(rec)) + + assert summary["recovered"] == 1 + assert not (staging / NAME).exists() + # The clip landed in the archive at its grouped destination. + dests = list(rec.rglob(NAME)) + assert len(dests) == 1 and dests[0].read_bytes() == b"b" * 20 + + +def test_recover_staging_keeps_fresh_part_removes_stale(env): + rec, db, staging = env + fresh = staging / (NAME + ".part") + fresh.write_bytes(b"streaming") + stale = staging / ("2026_0101_080000_0001F.MP4.part") + stale.write_bytes(b"dead upload") + old = time.time() - 7200 + os.utime(stale, (old, old)) + + importer.recover_staging(db, _snap(rec)) + + assert fresh.exists(), "recovery deleted an in-flight upload's .part" + assert not stale.exists(), "stale crashed-upload debris kept forever" + + +def test_folder_ingest_recovers_instead_of_deleting(env): + rec, db, staging = env + (staging / NAME).write_bytes(b"b" * 20) + root = rec.parent / "empty-import" + root.mkdir() + + importer.run_folder_ingest(db, _snap(rec), hub=None, loop=None, + root=str(root)) + + dests = list(rec.rglob(NAME)) + assert len(dests) == 1, \ + "folder ingest destroyed a staged clip instead of recovering it" + + +async def test_upload_streams_to_part_name(tmp_path, monkeypatch): + """While bytes are in flight the staging file must carry a .part + suffix — only a verified complete upload gets the plain name that + recovery treats as safe to archive.""" + from starlette.requests import Request + + from web.routers import imports as imports_router + + rec = tmp_path / "rec" + rec.mkdir() + snap = _snap(rec) + provider = MagicMock() + provider.get.return_value = snap + db = Database(str(tmp_path / "t.db")) + app = SimpleNamespace( + state=SimpleNamespace(settings_provider=provider, db=db)) + + staging = rec / importer.STAGING_DIRNAME + seen_during_stream: list[set] = [] + body = b"x" * 64 + messages = [ + {"type": "http.request", "body": body[:32], "more_body": True}, + {"type": "http.request", "body": body[32:], "more_body": False}, + ] + + async def receive(): + if staging.is_dir(): + seen_during_stream.append({p.name for p in staging.iterdir()}) + return messages.pop(0) + + scope = { + "type": "http", "method": "POST", "path": "/api/import/upload", + "query_string": b"", + "headers": [ + (b"x-import-path", NAME.encode()), + (b"x-import-size", str(len(body)).encode()), + ], + "app": app, + } + res = await imports_router.upload(Request(scope, receive)) + + assert res["status"] == "imported" + streamed_names = set().union(*seen_during_stream) if seen_during_stream else set() + plain = {n for n in streamed_names if not n.endswith(".part")} + assert plain == set(), \ + f"upload streamed to a plain staging name: {plain}" diff --git a/tests/test_login_ratelimit_eviction.py b/tests/test_login_ratelimit_eviction.py new file mode 100644 index 0000000..2c71adf --- /dev/null +++ b/tests/test_login_ratelimit_eviction.py @@ -0,0 +1,46 @@ +"""The login rate-limiter must not accumulate stale per-IP buckets. + +It keys on request.client.host; behind a reverse proxy that is one +shared IP (documented trade-off), but direct-LAN clients each get a +bucket — and buckets from IPs that never return used to linger +forever. Stale buckets are now swept. +""" +from __future__ import annotations + +import web.auth as auth_mod +from web.auth import LOGIN_MAX_ATTEMPTS, Auth + + +def _auth() -> Auth: + return Auth(password_hash="x", secret="s" * 32) + + +def test_rate_limit_still_blocks_within_window(monkeypatch): + now = [1000.0] + monkeypatch.setattr(auth_mod.time, "monotonic", lambda: now[0]) + a = _auth() + for _ in range(LOGIN_MAX_ATTEMPTS): + a.record_login_attempt("10.0.0.5") + import pytest + from fastapi import HTTPException + with pytest.raises(HTTPException): + a.record_login_attempt("10.0.0.5") + + +def test_stale_buckets_are_evicted(monkeypatch): + now = [1000.0] + monkeypatch.setattr(auth_mod.time, "monotonic", lambda: now[0]) + a = _auth() + + # 50 one-shot attempts from distinct IPs. + for i in range(50): + a.record_login_attempt(f"10.0.0.{i}") + assert len(a._login_attempts) == 50 + + # Time advances past the window; a single new attempt should sweep + # every now-stale bucket rather than leaving them to grow forever. + now[0] += auth_mod.LOGIN_WINDOW_SECONDS + 1 + a.record_login_attempt("10.0.1.1") + + assert len(a._login_attempts) == 1 + assert "10.0.1.1" in a._login_attempts diff --git a/tests/test_mqtt_backoff_reset.py b/tests/test_mqtt_backoff_reset.py new file mode 100644 index 0000000..110427b --- /dev/null +++ b/tests/test_mqtt_backoff_reset.py @@ -0,0 +1,98 @@ +"""MQTT reconnect backoff must reset after a stable connection, +discovery-cleanup state must be owned by the service (not poked in +from lifespan), and a timed-out ffprobe child must be reaped. +""" +from __future__ import annotations + +import asyncio +import time +from unittest.mock import MagicMock + +import aiomqtt + +from web.db import Database +from web.services import durations +from web.services.hub import Hub +from web.services.mqtt import MqttService + + +def _service(tmp_path, **snap_attrs) -> MqttService: + snap = MagicMock() + snap.mqtt_node_id = snap_attrs.get("node_id", "viofosync") + snap.mqtt_discovery_prefix = snap_attrs.get("prefix", "homeassistant") + snap.mqtt_host = "broker.local" + provider = MagicMock() + provider.get.return_value = snap + db = Database(str(tmp_path / "t.db")) + return MqttService(db=db, provider=provider, hub=Hub(), app=None) + + +# ---- backoff reset after a stable connection ---- + +async def test_backoff_resets_after_stable_connection(tmp_path, monkeypatch): + svc = _service(tmp_path) + monkeypatch.setattr(MqttService, "BACKOFF_STEPS", (0.01, 5.0)) + monkeypatch.setattr(MqttService, "STABLE_RESET_S", 0.05, raising=False) + + attempt_times: list[float] = [] + calls = {"n": 0} + + async def fake_connect(aiomqtt_mod, cfg): + calls["n"] += 1 + attempt_times.append(time.monotonic()) + if calls["n"] == 2: + # Stable connection: outlives STABLE_RESET_S, then drops. + await asyncio.sleep(0.1) + if calls["n"] >= 3: + svc._stop.set() + raise aiomqtt.MqttError("broker went away") + + monkeypatch.setattr(svc, "_connect_and_loop", fake_connect) + await asyncio.wait_for(svc._run(), timeout=5.0) + + assert calls["n"] >= 3 + # Attempt 1 fails instantly -> idx moves up the ladder. Attempt 2 + # was stable, so the delay before attempt 3 must be the FIRST + # step (0.01s), not the escalated 5s one. + gap = attempt_times[2] - (attempt_times[1] + 0.1) + assert gap < 1.0, ( + f"backoff not reset after a stable connection (waited {gap:.2f}s)" + ) + + +# ---- discovery cleanup state owned by the service ---- + +def test_last_topology_initialised_at_construction(tmp_path): + svc = _service(tmp_path, node_id="car2", prefix="ha") + assert svc._last_node_id == "car2" + assert svc._last_discovery_prefix == "ha" + + +# ---- ffprobe child reaped on timeout ---- + +async def test_ffprobe_timeout_kills_and_reaps_child(monkeypatch): + state = {"killed": False, "reaped": 0} + + class _HangProc: + def kill(self): + state["killed"] = True + + async def wait(self): + state["reaped"] += 1 + return -9 + + async def communicate(self): + await asyncio.sleep(60) + + async def fake_exec(*argv, **kwargs): + return _HangProc() + + monkeypatch.setattr(durations.shutil, "which", lambda n: "/bin/ffprobe") + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(durations, "_FFPROBE_TIMEOUT_S", 0.05, raising=False) + + got = await durations._probe_duration_ffprobe("/x.mp4") + + assert got is None + assert state["killed"], "timed-out ffprobe left running" + assert state["reaped"] == 1, "killed ffprobe never reaped (zombie)" diff --git a/tests/test_mqtt_password_masking.py b/tests/test_mqtt_password_masking.py new file mode 100644 index 0000000..16a68ea --- /dev/null +++ b/tests/test_mqtt_password_masking.py @@ -0,0 +1,104 @@ +"""MQTT_PASSWORD must be write-only over the settings API. + +GET /api/settings used to return the broker password in cleartext to +anything that could read one authenticated GET. It is now masked with +a sentinel; submitting the sentinel back means "unchanged", and the +Test-connection endpoint substitutes the stored password when handed +the sentinel. +""" +from __future__ import annotations + +import bcrypt +import pytest +from fastapi.testclient import TestClient + +from web import settings as settings_mod +from web.settings_schema import MASKED_SECRET + + +@pytest.fixture +def client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + data["MQTT_PASSWORD"] = "s3cr3t-broker-pw" + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + + class _FakeMqtt: + def __init__(self, **kw): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): return {"state": "idle", "detail": None, + "last_published_at": None} + + monkeypatch.setattr("web.app.MqttService", _FakeMqtt) + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + csrf = c.get("/api/auth/csrf").json()["csrf"] + c.headers["X-CSRF-Token"] = csrf + yield c + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def test_get_masks_existing_password(client): + body = client.get("/api/settings").json() + assert body["editable"]["MQTT_PASSWORD"] == MASKED_SECRET + assert "s3cr3t-broker-pw" not in str(body) + + +def test_put_sentinel_leaves_password_unchanged(client): + r = client.put("/api/settings", json={ + "MQTT_PASSWORD": MASKED_SECRET, "MQTT_USERNAME": "newuser", + }) + assert r.status_code == 200 + snap = settings_mod.get_provider().get() + assert snap.mqtt_password == "s3cr3t-broker-pw" # untouched + assert snap.mqtt_username == "newuser" + + +def test_put_real_value_updates_password(client): + r = client.put("/api/settings", json={"MQTT_PASSWORD": "brand-new-pw"}) + assert r.status_code == 200 + assert settings_mod.get_provider().get().mqtt_password == "brand-new-pw" + + +def test_put_empty_string_clears_password(client): + r = client.put("/api/settings", json={"MQTT_PASSWORD": ""}) + assert r.status_code == 200 + assert settings_mod.get_provider().get().mqtt_password == "" + + +def test_blank_password_is_not_masked(client): + settings_mod.get_provider().update({"MQTT_PASSWORD": ""}, actor="t") + body = client.get("/api/settings").json() + assert body["editable"]["MQTT_PASSWORD"] == "" + + +def test_test_endpoint_substitutes_stored_password_for_sentinel(client, monkeypatch): + captured = {} + import aiomqtt + + class _Spy: + def __init__(self, **kw): + captured.update(kw) + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + + monkeypatch.setattr(aiomqtt, "Client", _Spy) + r = client.post("/api/mqtt/test", json={ + "host": "broker.local", "password": MASKED_SECRET, + }) + assert r.status_code == 200 + assert captured["password"] == "s3cr3t-broker-pw", \ + "test endpoint did not substitute the stored password for the sentinel" diff --git a/tests/test_mqtt_reconnect_classification.py b/tests/test_mqtt_reconnect_classification.py new file mode 100644 index 0000000..6767087 --- /dev/null +++ b/tests/test_mqtt_reconnect_classification.py @@ -0,0 +1,57 @@ +"""Reconnect vs. fatal-error classification in MqttService._run. + +_connect_and_loop runs its workers inside an asyncio.TaskGroup, which +reports any task failure as an ExceptionGroup rather than the bare +exception. A routine broker disconnect (aiomqtt.MqttError, e.g. +"Disconnected during message iteration") therefore reaches _run wrapped +in a group. It must be classified as RECONNECTING, not logged as an +unexpected fatal ERROR. +""" +from __future__ import annotations + +import asyncio + +import aiomqtt + +from web.services.mqtt import ConnState, MqttService + + +def _make_service(raises: BaseException) -> MqttService: + svc = MqttService(db=None, provider=None, hub=None, app=None) + svc._stop = asyncio.Event() + svc._cfg = lambda: {"host": "broker", "port": 1883} + + async def _boom(_aiomqtt_mod, _cfg): + # Break out of the while loop after this single attempt so the + # backoff sleep is skipped (mirrors _run's `if _stop: break`). + svc._stop.set() + raise raises + + svc._connect_and_loop = _boom # type: ignore[assignment] + return svc + + +def test_taskgroup_disconnect_is_reconnecting(): + eg = ExceptionGroup( + "unhandled errors in a TaskGroup (1 sub-exception)", + [aiomqtt.MqttError("Disconnected during message iteration")], + ) + svc = _make_service(eg) + asyncio.run(svc._run()) + assert svc._state is ConnState.RECONNECTING + assert "Disconnected during message iteration" in (svc._detail or "") + + +def test_bare_mqtterror_is_reconnecting(): + svc = _make_service(aiomqtt.MqttError("Operation timed out")) + asyncio.run(svc._run()) + assert svc._state is ConnState.RECONNECTING + assert "Operation timed out" in (svc._detail or "") + + +def test_genuine_error_in_group_is_error(): + eg = ExceptionGroup("boom", [ValueError("something truly unexpected")]) + svc = _make_service(eg) + asyncio.run(svc._run()) + assert svc._state is ConnState.ERROR + assert "something truly unexpected" in (svc._detail or "") diff --git a/tests/test_mqtt_state_offloop.py b/tests/test_mqtt_state_offloop.py new file mode 100644 index 0000000..4cbe43c --- /dev/null +++ b/tests/test_mqtt_state_offloop.py @@ -0,0 +1,59 @@ +"""MQTT state_fn calls must not starve the event loop. + +state_disk_used does a full archive tree walk in quota mode — on a +NAS that's seconds of blocking I/O, invoked from the 60s tick and +every full-state publish. It has to run off the loop. +""" +from __future__ import annotations + +import asyncio +import time +from unittest.mock import MagicMock, patch + +from web.db import Database +from web.services import mqtt_topology as topo +from web.services.hub import Hub +from web.services.mqtt import MqttService + + +class _Client: + async def publish(self, *args, **kwargs) -> None: + pass + + +def _entity(state_fn) -> topo.EntityDef: + return topo.EntityDef( + object_id="disk_used", component="sensor", name="Disk used", + icon=None, device_class=None, state_class=None, + unit_of_measurement=None, enabled_by_default=True, + min_publish_interval_s=0.0, state_fn=state_fn, + command_handler=None, + ) + + +async def test_slow_state_fn_does_not_starve_loop(tmp_path): + db = Database(str(tmp_path / "t.db")) + svc = MqttService(db=db, provider=MagicMock(), hub=Hub(), app=None) + + def slow_state(hub, db_, snap): + time.sleep(0.3) # simulate the quota-mode archive walk + return "42" + + ticks = 0 + + async def _ticker(): + nonlocal ticks + while True: + await asyncio.sleep(0.02) + ticks += 1 + + t = asyncio.create_task(_ticker()) + try: + with patch.object(topo, "TOPOLOGY", (_entity(slow_state),)): + await svc._publish_full_state( + _Client(), {"node_id": "n", "qos": 0}, + ) + finally: + t.cancel() + + assert ticks >= 5, f"event loop starved by state_fn ({ticks} ticks)" diff --git a/tests/test_mvhd_probe.py b/tests/test_mvhd_probe.py new file mode 100644 index 0000000..49d7df8 --- /dev/null +++ b/tests/test_mvhd_probe.py @@ -0,0 +1,160 @@ +"""Tests for the direct mvhd-box duration parser. + +Reading the MP4 ``moov/mvhd`` box gives clip duration without spawning an +ffprobe subprocess per clip — far cheaper for the duration sweep across a +multi-thousand-clip archive. The parser seeks past ``mdat`` rather than +reading it, so it's cheap even when ``moov`` sits at the end of a large +file (the usual dashcam layout). Anything it can't parse returns None so +the caller falls back to ffprobe. +""" +from __future__ import annotations + +import shutil +import struct +import subprocess +from pathlib import Path + +import pytest + +from web.services import durations + +# --- ISO-BMFF box builders for deterministic fixtures --- + +def _box(btype: bytes, payload: bytes) -> bytes: + return struct.pack(">I", 8 + len(payload)) + btype + payload + + +def _box64(btype: bytes, payload: bytes) -> bytes: + # size==1 sentinel, then a 64-bit largesize (how big mdat is encoded) + return struct.pack(">I", 1) + btype + struct.pack(">Q", 16 + len(payload)) + payload + + +def _mvhd_v0(timescale: int, duration: int) -> bytes: + p = bytes([0, 0, 0, 0]) # version 0 + flags + p += struct.pack(">I", 0) # creation_time + p += struct.pack(">I", 0) # modification_time + p += struct.pack(">I", timescale) + p += struct.pack(">I", duration) + p += b"\x00" * 80 # trailing fields (unparsed) + return _box(b"mvhd", p) + + +def _mvhd_v1(timescale: int, duration: int) -> bytes: + p = bytes([1, 0, 0, 0]) # version 1 + flags + p += struct.pack(">Q", 0) # creation_time (64-bit) + p += struct.pack(">Q", 0) # modification_time (64-bit) + p += struct.pack(">I", timescale) + p += struct.pack(">Q", duration) # duration (64-bit) + p += b"\x00" * 80 + return _box(b"mvhd", p) + + +def _w(tmp_path: Path, name: str, data: bytes) -> str: + p = tmp_path / name + p.write_bytes(data) + return str(p) + + +def test_mvhd_v0_moov_first(tmp_path: Path) -> None: + data = _box(b"ftyp", b"isomiso2") + _box(b"moov", _mvhd_v0(1000, 5000)) + assert durations._probe_duration_mvhd(_w(tmp_path, "a.mp4", data)) == pytest.approx(5.0) + + +def test_mvhd_after_large_mdat(tmp_path: Path) -> None: + """Dashcam layout: moov AFTER a big mdat. Parser must seek past mdat + (not read it) and still find mvhd.""" + data = (_box(b"ftyp", b"isom") + + _box(b"mdat", b"\x00" * 100_000) + + _box(b"moov", _mvhd_v0(90000, 90000 * 12))) + assert durations._probe_duration_mvhd(_w(tmp_path, "b.mp4", data)) == pytest.approx(12.0) + + +def test_mvhd_v1_64bit_duration(tmp_path: Path) -> None: + data = _box(b"ftyp", b"isom") + _box(b"moov", _mvhd_v1(48000, 48000 * 7)) + assert durations._probe_duration_mvhd(_w(tmp_path, "c.mp4", data)) == pytest.approx(7.0) + + +def test_mvhd_64bit_mdat_size(tmp_path: Path) -> None: + """Large mdat encoded with the 64-bit size form must be skipped correctly.""" + data = (_box(b"ftyp", b"isom") + + _box64(b"mdat", b"\x00" * 5000) + + _box(b"moov", _mvhd_v0(1000, 3000))) + assert durations._probe_duration_mvhd(_w(tmp_path, "d.mp4", data)) == pytest.approx(3.0) + + +def test_mvhd_skips_empty_free_box(tmp_path: Path) -> None: + """ffmpeg emits a zero-payload ``free`` box (size 8) before mdat; the + scan must treat it as valid and keep going, not bail.""" + data = (_box(b"ftyp", b"isom") + + struct.pack(">I", 8) + b"free" # empty free box + + _box(b"mdat", b"\x00" * 200) + + _box(b"moov", _mvhd_v0(1000, 4000))) + assert durations._probe_duration_mvhd(_w(tmp_path, "free.mp4", data)) == pytest.approx(4.0) + + +def test_mvhd_no_moov_returns_none(tmp_path: Path) -> None: + data = _box(b"ftyp", b"isom") + _box(b"mdat", b"\x00" * 1000) + assert durations._probe_duration_mvhd(_w(tmp_path, "e.mp4", data)) is None + + +def test_mvhd_unknown_duration_returns_none(tmp_path: Path) -> None: + data = _box(b"ftyp", b"isom") + _box(b"moov", _mvhd_v0(1000, 0xFFFFFFFF)) + assert durations._probe_duration_mvhd(_w(tmp_path, "f.mp4", data)) is None + + +def test_mvhd_zero_timescale_returns_none(tmp_path: Path) -> None: + data = _box(b"ftyp", b"isom") + _box(b"moov", _mvhd_v0(0, 5000)) + assert durations._probe_duration_mvhd(_w(tmp_path, "z.mp4", data)) is None + + +def test_mvhd_missing_file_returns_none(tmp_path: Path) -> None: + assert durations._probe_duration_mvhd(str(tmp_path / "nope.mp4")) is None + + +def test_mvhd_truncated_returns_none(tmp_path: Path) -> None: + # moov header claims a size the file doesn't contain + data = _box(b"ftyp", b"isom") + struct.pack(">I", 0x1000) + b"moov\x00\x00" + assert durations._probe_duration_mvhd(_w(tmp_path, "g.mp4", data)) is None + + +def test_mvhd_matches_real_ffmpeg_clip(tmp_path: Path) -> None: + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + pytest.skip("ffmpeg not available") + clip = tmp_path / "real.mp4" + subprocess.run( + [ffmpeg, "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", + "-i", "testsrc=size=320x180:duration=5:rate=30", "-c:v", "libx264", + str(clip)], + check=True, + ) + assert durations._probe_duration_mvhd(str(clip)) == pytest.approx(5.0, abs=0.3) + + +# --- orchestration: probe_duration prefers mvhd, falls back to ffprobe --- + + +async def test_probe_duration_prefers_mvhd_over_ffprobe(tmp_path: Path, monkeypatch) -> None: + data = _box(b"ftyp", b"isom") + _box(b"moov", _mvhd_v0(1000, 8000)) + path = _w(tmp_path, "h.mp4", data) + + def _boom(*a, **k): + raise AssertionError("ffprobe must not run when mvhd parses") + monkeypatch.setattr(durations.asyncio, "create_subprocess_exec", _boom) + + assert await durations.probe_duration(path) == pytest.approx(8.0) + + +async def test_probe_duration_falls_back_to_ffprobe(tmp_path: Path, monkeypatch) -> None: + # No parseable moov -> mvhd returns None -> ffprobe is used. + path = _w(tmp_path, "i.mp4", _box(b"ftyp", b"isom")) + + class _P: + async def communicate(self): + return (b"33.3\n", b"") + async def fake_exec(*a, **k): + return _P() + monkeypatch.setattr(durations.shutil, "which", lambda _n: "/usr/bin/ffprobe") + monkeypatch.setattr(durations.asyncio, "create_subprocess_exec", fake_exec) + + assert await durations.probe_duration(path) == pytest.approx(33.3) diff --git a/tests/test_progress_origin.py b/tests/test_progress_origin.py new file mode 100644 index 0000000..e77bca9 --- /dev/null +++ b/tests/test_progress_origin.py @@ -0,0 +1,57 @@ +"""Cross-site WebSocket hijacking guard on /api/progress. + +Browsers send cookies on cross-origin WS handshakes and WS is exempt +from same-origin fetch rules — without an Origin check, any page the +logged-in admin visits can open the socket and read the live event +stream. Non-browser clients (curl, HA) send no Origin and must still +be allowed; auth alone gates those. +""" +from __future__ import annotations + +from types import SimpleNamespace + +from web.routers.progress import progress + + +class _Auth: + def validate_session(self, token) -> bool: + return False # force the 4401 path when auth is reached + + +class _WS: + def __init__(self, *, origin: str | None, host: str = "nas:8080"): + self.headers = {"host": host} + if origin is not None: + self.headers["origin"] = origin + self.cookies: dict = {} + self.closed: list = [] + self.app = SimpleNamespace(state=SimpleNamespace(auth=_Auth())) + + async def close(self, code: int = 1000) -> None: + self.closed.append(code) + + +async def test_cross_origin_handshake_rejected(): + ws = _WS(origin="http://evil.example") + await progress(ws) + assert ws.closed == [4403], \ + f"cross-origin WS not rejected with 4403 (got {ws.closed})" + + +async def test_same_origin_proceeds_to_auth(): + ws = _WS(origin="http://nas:8080") + await progress(ws) + assert ws.closed == [4401] # passed origin, failed (fake) auth + + +async def test_no_origin_header_proceeds_to_auth(): + # curl / Home Assistant / scripts send no Origin. + ws = _WS(origin=None) + await progress(ws) + assert ws.closed == [4401] + + +async def test_origin_with_https_scheme_and_same_host_allowed(): + ws = _WS(origin="https://nas:8080") + await progress(ws) + assert ws.closed == [4401] diff --git a/tests/test_protocol_cancel.py b/tests/test_protocol_cancel.py index 46636bb..7f88e0f 100644 --- a/tests/test_protocol_cancel.py +++ b/tests/test_protocol_cancel.py @@ -84,3 +84,36 @@ def cancel_check(): assert get_opens["n"] == 1 # No half-written .part file left behind. assert list(tmp_path.glob("*.part")) == [] + + +# ---- cancellation during retry backoff ---- + +def test_cancel_honoured_during_retry_backoff(tmp_path, monkeypatch): + """A pause/stop/unreachable signal must interrupt the inter-attempt + backoff sleep, not wait out the full 5-50s ladder.""" + import datetime + import time + from unittest.mock import patch + + from viofosync_lib import DownloadCancelled, _protocol + from viofosync_lib._archive import Recording + + monkeypatch.setattr(_protocol, "max_download_attempts", 3) + monkeypatch.setattr(_protocol, "RETRY_BACKOFF", 30) # would be a long wait + + rec = Recording( + "2026_0101_120000_0001F.MP4", "/DCIM/Movie/x.MP4", 1000, 0, + datetime.datetime(2026, 1, 1, 12, 0), 0, + ) + + def fail(url_or_req, *a, **k): + raise ConnectionRefusedError("transient") + + started = time.monotonic() + with patch("urllib.request.urlopen", side_effect=fail): + with __import__("pytest").raises(DownloadCancelled): + _protocol.download_file( + "http://192.0.2.1", rec, str(tmp_path), "", + cancel_check=lambda: True, + ) + assert time.monotonic() - started < 2.0, "backoff sleep ignored cancel_check" diff --git a/tests/test_protocol_listing_timeout.py b/tests/test_protocol_listing_timeout.py new file mode 100644 index 0000000..155fdef --- /dev/null +++ b/tests/test_protocol_listing_timeout.py @@ -0,0 +1,53 @@ +"""The XML listing request must carry a socket timeout. + +Without one, a half-open dashcam connection (Wi-Fi drop mid-handshake) +blocks the sync worker's executor thread forever — the HTML-scrape +path already passes ``socket_timeout``; the XML path must too. +""" +from __future__ import annotations + +from unittest.mock import patch + +from viofosync_lib import _protocol + + +class _FakeResponse: + def getcode(self) -> int: + return 200 + + def read(self) -> bytes: + return b"" + + +def test_xml_listing_passes_socket_timeout(): + captured: dict = {} + + def fake_urlopen(request, *args, **kwargs): + captured["timeout"] = kwargs.get("timeout") + return _FakeResponse() + + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + recs = _protocol.get_dashcam_filenames("http://192.0.2.1") + + assert recs == [] + assert captured["timeout"] == _protocol.socket_timeout + + +def test_xml_listing_timeout_tracks_module_setting(): + """download_file_with temporarily overrides the module global — + the listing must honour the value at call time, not import time.""" + captured: dict = {} + + def fake_urlopen(request, *args, **kwargs): + captured["timeout"] = kwargs.get("timeout") + return _FakeResponse() + + saved = _protocol.socket_timeout + try: + _protocol.socket_timeout = 7.5 + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + _protocol.get_dashcam_filenames("http://192.0.2.1") + finally: + _protocol.socket_timeout = saved + + assert captured["timeout"] == 7.5 diff --git a/tests/test_queue_changed_events.py b/tests/test_queue_changed_events.py new file mode 100644 index 0000000..69fc06d --- /dev/null +++ b/tests/test_queue_changed_events.py @@ -0,0 +1,60 @@ +"""queue_changed broadcasts must survive any caller context. + +The prioritize/retry routes are sync ``def`` handlers running in the +threadpool — no running loop, no ``loop`` argument — and their +queue_changed events used to be silently dropped, leaving every +client's queue badges stale until the next worker cycle. +""" +from __future__ import annotations + +import asyncio + +import pytest + +from web.db import Database +from web.services import queue as q +from web.services.hub import Hub + + +@pytest.fixture +def db(tmp_path) -> Database: + return Database(str(tmp_path / "t.db")) + + +async def test_emit_from_threadpool_reaches_hub(db): + hub = Hub() + hub.bind_loop(asyncio.get_running_loop()) + + received: list = [] + + async def _spy(event): + received.append(event) + + hub.broadcast = _spy # type: ignore[assignment] + + # Threadpool context: no running loop, no loop kwarg — exactly + # how the sync queue routes call it. + await asyncio.to_thread(q.emit_queue_changed, db, hub) + await asyncio.sleep(0.05) # let the scheduled coroutine run + + assert any(e.get("type") == "queue_changed" for e in received), \ + "queue_changed dropped when emitted off-loop without a loop arg" + + +async def test_schedule_broadcast_falls_back_to_bound_loop(): + hub = Hub() + hub.bind_loop(asyncio.get_running_loop()) + + received: list = [] + + async def _spy(event): + received.append(event) + + hub.broadcast = _spy # type: ignore[assignment] + + await asyncio.to_thread( + hub.schedule_broadcast, None, {"type": "ping"} + ) + await asyncio.sleep(0.05) + + assert received == [{"type": "ping"}] diff --git a/tests/test_reconcile_present.py b/tests/test_reconcile_present.py new file mode 100644 index 0000000..22fa9f5 --- /dev/null +++ b/tests/test_reconcile_present.py @@ -0,0 +1,55 @@ +"""Reconcile must not leave on-disk clips stuck as pending/failed. + +Regression: a clip the dashcam listed (queued pending), then placed on +disk by another path (bulk web-upload / manual copy), used to stay +pending because reconcile only inserted a 'done' row when the filename +was absent from the queue. On the next Wi-Fi cycle the worker re-tried +the download and the dashcam 404'd it. +""" +from __future__ import annotations + +import datetime as _dt +from pathlib import Path + +import pytest + +from web.db import Database +from web.services import queue as q + + +class _Rec: + def __init__(self, filename: str, *, filepath: str = "/DCIM/Movie", + size: int = 1000) -> None: + self.filename = filename + self.filepath = filepath + self.size = size + self.datetime = _dt.datetime(2026, 5, 19, 7, 47, 52) + + +@pytest.fixture +def db(tmp_path: Path) -> Database: + return Database(str(tmp_path / ".viofosync.db")) + + +def _states(db: Database) -> dict[str, str]: + with db.conn() as c: + return {r["filename"]: r["state"] for r in c.execute( + "SELECT filename, state FROM download_queue").fetchall()} + + +def _seed(db: Database, filename: str, state: str) -> None: + with db.write() as c: + c.execute( + "INSERT INTO download_queue (filename, source_dir, state, " + "enqueued_at) VALUES (?,?,?,0)", + (filename, "/DCIM/Movie", state), + ) + + +@pytest.mark.parametrize("state", ["pending", "failed"]) +def test_reconcile_heals_on_disk_clip_stuck_in_queue(db: Database, state: str): + name = "2026_0519_074752_022262PF.MP4" + _seed(db, name, state) # camera listed it earlier + # File is now on disk (web-upload) and the camera still lists it. + q.reconcile(db, [_Rec(name)], present_filenames=[name]) + assert _states(db)[name] == "done" diff --git a/tests/test_refresh_listing.py b/tests/test_refresh_listing.py index dbbbeee..2642732 100644 --- a/tests/test_refresh_listing.py +++ b/tests/test_refresh_listing.py @@ -147,3 +147,42 @@ def _boom(): assert ok is False # Queue is untouched. assert _queue_rows(db) == [] + + +# ---- event-loop responsiveness ---- + +async def test_refresh_does_not_block_event_loop(db: Database) -> None: + """The NAS directory walk + queue reconcile are blocking I/O and + must run off the loop — a slow recordings volume used to freeze + every HTTP request and WebSocket for its duration.""" + import asyncio + import time + + provider = MagicMock() + provider.get.return_value = _make_snap() + worker = SyncWorker(db, provider, Hub()) + + def _slow_present(): + time.sleep(0.3) # simulate a slow NAS walk + return [] + + ticks = 0 + + async def _ticker(): + nonlocal ticks + while True: + await asyncio.sleep(0.02) + ticks += 1 + + t = asyncio.create_task(_ticker()) + try: + with patch.object(worker, "_fetch_listing", return_value=[_Rec("A.MP4")]), \ + patch.object(worker, "_present_filenames", side_effect=_slow_present): + await worker._refresh_listing_and_reconcile() + finally: + t.cancel() + + # With the walk on the loop the ticker barely runs (~0-2 ticks); + # off the loop it accumulates ~15. Threshold splits them cleanly. + assert ticks >= 5, f"event loop starved during reconcile ({ticks} ticks)" + assert _queue_rows(db) == [("A.MP4", "pending")] diff --git a/tests/test_retention_export_guard.py b/tests/test_retention_export_guard.py new file mode 100644 index 0000000..8659647 --- /dev/null +++ b/tests/test_retention_export_guard.py @@ -0,0 +1,173 @@ +"""Retention must not delete clips an export job will read. + +Exports of old footage are the norm (you export *before* retention +takes it); the sweep deleting a source file mid-render fails the job +with ENOENT on the next segment. Pending/active export jobs publish a +protect-set the sweep and make_room_for must honour. +""" +from __future__ import annotations + +import json +import time as _time +from collections import namedtuple +from pathlib import Path + +import pytest + +from web.db import Database +from web.services import retention as ret +from web.services.exporter import export_protect_ids + +DiskUsage = namedtuple("DiskUsage", ["total", "used", "free"]) + + +@pytest.fixture +def env(tmp_path: Path): + rec = tmp_path / "rec" + rec.mkdir() + db = Database(str(rec / ".viofosync.db")) + return rec, db + + +def _make_clip(rec: Path, db: Database, *, basename: str, ts: int) -> int: + day = _time.strftime("%Y-%m-%d", _time.gmtime(ts)) + folder = rec / day + folder.mkdir(exist_ok=True) + path = folder / basename + path.write_bytes(b"x" * 1024) + with db.write() as c: + cur = c.execute( + "INSERT INTO clip_index " + "(path, basename, group_name, timestamp, camera, sequence, " + " event_type, size_bytes, has_gpx, scanned_at) " + "VALUES (?, ?, ?, ?, 'F', 1, 'normal', 1024, 0, ?)", + (str(path), basename, day, ts, int(_time.time())), + ) + return cur.lastrowid + + +def _add_job(db: Database, *, state: str, payload, clip_start=None, + clip_end=None) -> None: + with db.write() as c: + c.execute( + "INSERT INTO export_jobs (type, clip_ids, state, created_at, " + "clip_start, clip_end) VALUES ('join_front', ?, ?, 0, ?, ?)", + (json.dumps(payload), state, clip_start, clip_end), + ) + + +# ---- export_protect_ids payload forms ---- + +def test_protect_ids_dict_payload(env): + rec, db = env + _add_job(db, state="queued", payload={"clip_ids": [3, 7], "encoder": "software"}) + assert export_protect_ids(db) == frozenset({3, 7}) + + +def test_protect_ids_legacy_list_payload(env): + rec, db = env + _add_job(db, state="running", payload=[4, 5]) + assert export_protect_ids(db) == frozenset({4, 5}) + + +def test_protect_ids_timeline_job_uses_time_range(env): + rec, db = env + inside = _make_clip(rec, db, basename="IN.MP4", ts=5000) + outside = _make_clip(rec, db, basename="OUT.MP4", ts=50_000) + _add_job(db, state="paused", payload={"segments": [], "encoder": "software"}, + clip_start=4900, clip_end=6000) + ids = export_protect_ids(db) + assert inside in ids + assert outside not in ids + + +def test_protect_ids_ignores_finished_jobs(env): + rec, db = env + _add_job(db, state="done", payload={"clip_ids": [1]}) + _add_job(db, state="failed", payload={"clip_ids": [2]}) + assert export_protect_ids(db) == frozenset() + + +# ---- sweep honours the protect set ---- + +def test_disk_pressure_skips_protected_clips(env, monkeypatch): + rec, db = env + protected = _make_clip(rec, db, basename="A.MP4", ts=100) # oldest + _make_clip(rec, db, basename="B.MP4", ts=200) + _make_clip(rec, db, basename="C.MP4", ts=300) + + state = {"used_pct": 95} + monkeypatch.setattr( + "web.services.retention.shutil.disk_usage", + lambda p: DiskUsage(100, state["used_pct"], 100 - state["used_pct"]), + ) + orig = ret._delete_clip_files + + def wrapped(*a, **kw): + out = orig(*a, **kw) + state["used_pct"] -= 20 + return out + + monkeypatch.setattr(ret, "_delete_clip_files", wrapped) + + ret.sweep( + db, str(rec), max_days=0, disk_pct=80, protect_ro=True, + protect_ids=frozenset({protected}), _now=86400 * 365, + ) + + with db.conn() as c: + remaining = {r["basename"] for r in + c.execute("SELECT basename FROM clip_index")} + assert "A.MP4" in remaining, "sweep deleted a clip an export is reading" + assert "B.MP4" not in remaining # pressure satisfied by next-oldest + + +def test_time_rule_skips_protected_clips(env): + rec, db = env + now = 86400 * 365 + protected = _make_clip(rec, db, basename="OLD.MP4", ts=0) + _make_clip(rec, db, basename="OLD2.MP4", ts=1) + + summary = ret.sweep( + db, str(rec), max_days=30, disk_pct=0, protect_ro=True, + protect_ids=frozenset({protected}), _now=now, + ) + assert summary["deleted_time"] == 1 + with db.conn() as c: + remaining = {r["basename"] for r in + c.execute("SELECT basename FROM clip_index")} + assert remaining == {"OLD.MP4"} + + +# ---- make_room_for honours the protect set ---- + +def test_make_room_for_skips_protected_clips(env, monkeypatch): + rec, db = env + protected = _make_clip(rec, db, basename="A.MP4", ts=100) + _make_clip(rec, db, basename="B.MP4", ts=200) + + # Quota mode: pretend each clip is 0.6 GiB so one eviction fixes + # it; deletes must report the same fake size for the bookkeeping. + gib = 1 << 30 + monkeypatch.setattr( + ret, "_scan_dir_bytes", + lambda p, exclude=frozenset(): + len(list(Path(p).rglob("*.MP4"))) * int(0.6 * gib), + ) + orig_del = ret._delete_clip_files + + def _del(*a, **kw): + orig_del(*a, **kw) + return int(0.6 * gib) + + monkeypatch.setattr(ret, "_delete_clip_files", _del) + ok = ret.make_room_for( + db, str(rec), size=1, before_ts=10_000, disk_pct=0, quota_gb=1, + protect_ro=True, protect_ids=frozenset({protected}), + ) + assert ok is True + with db.conn() as c: + remaining = {r["basename"] for r in + c.execute("SELECT basename FROM clip_index")} + assert "A.MP4" in remaining + assert "B.MP4" not in remaining diff --git a/tests/test_route_cache.py b/tests/test_route_cache.py new file mode 100644 index 0000000..a0f5065 --- /dev/null +++ b/tests/test_route_cache.py @@ -0,0 +1,59 @@ +"""Tests for the per-day route aggregation cache.""" +from __future__ import annotations + +import os +from pathlib import Path + +from web.services import route_cache + + +def test_signature_is_order_independent(tmp_path: Path): + a = tmp_path / "1.gpx" + a.write_text("x") + b = tmp_path / "2.gpx" + b.write_text("yy") + assert route_cache.signature([str(a), str(b)]) == \ + route_cache.signature([str(b), str(a)]) + + +def test_signature_changes_when_a_file_changes(tmp_path: Path): + a = tmp_path / "1.gpx" + a.write_text("x") + before = route_cache.signature([str(a)]) + a.write_text("xxxxxxxx") # size changes + os.utime(a, (a.stat().st_atime, a.stat().st_mtime + 10)) # and mtime + assert route_cache.signature([str(a)]) != before + + +def test_signature_changes_when_a_file_is_added(tmp_path: Path): + a = tmp_path / "1.gpx" + a.write_text("x") + one = route_cache.signature([str(a)]) + b = tmp_path / "2.gpx" + b.write_text("y") + assert route_cache.signature([str(a), str(b)]) != one + + +def test_signature_ignores_missing_files(tmp_path: Path): + a = tmp_path / "1.gpx" + a.write_text("x") + missing = str(tmp_path / "nope.gpx") + assert route_cache.signature([str(a), missing]) == \ + route_cache.signature([str(a)]) + + +def test_store_then_load_roundtrips(tmp_path: Path): + rec = str(tmp_path) + payload = {"date": "2026-06-02", "point_count": 3, "journeys": []} + route_cache.store(rec, "2026-06-02", "sig1", payload) + assert route_cache.load(rec, "2026-06-02", "sig1") == payload + + +def test_load_returns_none_on_signature_mismatch(tmp_path: Path): + rec = str(tmp_path) + route_cache.store(rec, "2026-06-02", "sig1", {"point_count": 1}) + assert route_cache.load(rec, "2026-06-02", "sig2") is None + + +def test_load_returns_none_when_absent(tmp_path: Path): + assert route_cache.load(str(tmp_path), "2026-06-02", "sig") is None diff --git a/tests/test_route_endpoint.py b/tests/test_route_endpoint.py new file mode 100644 index 0000000..97a8558 --- /dev/null +++ b/tests/test_route_endpoint.py @@ -0,0 +1,85 @@ +"""Tests for GET /api/archive/day/{date}/route aggregation caching.""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from web.routers import archive + + +@pytest.fixture +def authed_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + from web import app as app_mod + from web import settings as settings_mod + monkeypatch.setenv("VIOFOSYNC_RESTART_DISABLED", "1") + settings_mod.reset_for_tests() + application = app_mod.create_app() + with TestClient(application) as c: + c.post("/setup", data={ + "address": "192.168.1.230", + "password": "twelve-chars-min!", + "confirm": "twelve-chars-min!", + }) + csrf = c.get("/api/auth/csrf").json()["csrf"] + c.headers.update({"x-csrf-token": csrf}) + yield c + + +def _add_gpx_clip(app, rec: Path, clip_id: int, date: str = "2026-06-02") -> str: + day_dir = rec / date + day_dir.mkdir(parents=True, exist_ok=True) + mp4 = day_dir / f"{clip_id}.MP4" + mp4.write_bytes(b"\0") + gpx = day_dir / f"{clip_id}.MP4.gpx" + gpx.write_text("") + with app.state.db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(id, path, basename, group_name, timestamp, camera, sequence, " + " event_type, has_gpx, gps_examined, scanned_at) " + "VALUES (?,?,?,?,?,?,?,?,1,0,?)", + (clip_id, str(mp4), mp4.name, date, 1_717_312_440 + clip_id, + "F", clip_id, "normal", 1_717_312_440), + ) + return str(gpx) + + +def _counting_aggregate(calls): + def _agg(paths): + calls["n"] += 1 + return [], [], [] + return _agg + + +def test_route_aggregation_is_cached(authed_client, tmp_recordings_dir, monkeypatch): + app = authed_client.app + _add_gpx_clip(app, tmp_recordings_dir, 1) + calls = {"n": 0} + monkeypatch.setattr(archive.gps_service, "aggregate_day", + _counting_aggregate(calls)) + + r1 = authed_client.get("/api/archive/day/2026-06-02/route") + r2 = authed_client.get("/api/archive/day/2026-06-02/route") + assert r1.status_code == 200 and r2.status_code == 200 + assert r1.json() == r2.json() + assert calls["n"] == 1 # second request served from cache + + +def test_route_cache_busts_when_gpx_changes( + authed_client, tmp_recordings_dir, monkeypatch +): + app = authed_client.app + gpx = _add_gpx_clip(app, tmp_recordings_dir, 1) + calls = {"n": 0} + monkeypatch.setattr(archive.gps_service, "aggregate_day", + _counting_aggregate(calls)) + + authed_client.get("/api/archive/day/2026-06-02/route") + Path(gpx).write_text("changed-and-larger") + st = Path(gpx).stat() + os.utime(gpx, (st.st_atime, st.st_mtime + 10)) + authed_client.get("/api/archive/day/2026-06-02/route") + assert calls["n"] == 2 # changed GPX -> recomputed diff --git a/tests/test_scanner_prune.py b/tests/test_scanner_prune.py new file mode 100644 index 0000000..693acc5 --- /dev/null +++ b/tests/test_scanner_prune.py @@ -0,0 +1,82 @@ +"""scanner.scan must not wipe the index when recordings is unavailable. + +Root cause of "the duration sweep re-runs across all clips after it +completed yesterday": scanner.scan rebuilds the index from a glob of the +recordings directory and prunes any DB row whose file it didn't see. When +that glob returns nothing — the volume not yet mounted at container start, +or a transient NAS glitch — the prune ran an unconditional +``DELETE FROM clip_index``, wiping every row. The next scan re-inserted +them via an INSERT that omits duration_s (→ NULL) and resets gps_examined, +so the duration sweep (and GPS re-exam, thumbs) re-ran across the whole +archive. +""" +from __future__ import annotations + +import logging +from pathlib import Path + +from web.db import Database +from web.services import scanner + + +def _insert(db: Database, path: str, *, duration_s: float = 42.0, + gps_examined: int = 1) -> None: + with db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(path, basename, group_name, timestamp, camera, sequence, " + " event_type, size_bytes, has_gpx, gps_examined, duration_s, " + " scanned_at) " + "VALUES (?, ?, '2026-06-03', 0, 'F', 1, 'normal', 100, 0, ?, ?, 0)", + (path, path.split("/")[-1], gps_examined, duration_s), + ) + + +def _counts(db: Database) -> tuple[int, int]: + with db.conn() as c: + row = c.execute( + "SELECT COUNT(*) AS n, " + " SUM(CASE WHEN duration_s > 0 THEN 1 ELSE 0 END) AS d " + "FROM clip_index" + ).fetchone() + return row["n"], (row["d"] or 0) + + +def test_empty_scan_does_not_wipe_index(tmp_path: Path, caplog) -> None: + """A scan that finds zero clips (recordings unavailable) must keep the + existing rows and their durations, not delete the whole index.""" + db = Database(str(tmp_path / "t.db")) + _insert(db, "/recordings/2026-06-03/2026_0603_082421_0001F.MP4") + _insert(db, "/recordings/2026-06-03/2026_0603_082421_0001R.MP4") + assert _counts(db) == (2, 2) + + empty = tmp_path / "recordings" + empty.mkdir() # exists but contains no clips -> glob yields nothing + + with caplog.at_level(logging.WARNING, logger="viofosync.scanner"): + scanner.scan(db, str(empty), "daily") + + assert _counts(db) == (2, 2) # index intact, durations preserved + assert any("skip" in r.getMessage().lower() or "0 clip" in r.getMessage() + for r in caplog.records), "expected a warning about the empty scan" + + +def test_scan_still_prunes_genuinely_vanished_file(tmp_path: Path) -> None: + """The empty-scan guard must not disable legitimate pruning: when the + scan DOES find clips, a row whose file is gone is still removed.""" + db = Database(str(tmp_path / "t.db")) + day = tmp_path / "recordings" / "2026-06-03" + day.mkdir(parents=True) + present = day / "2026_0603_082421_0001F.MP4" + present.write_bytes(b"\x00" * 16) + + _insert(db, str(present)) # on disk + _insert(db, "/recordings/2026-06-03/2026_0603_090000_0002F.MP4") # gone + + scanner.scan(db, str(tmp_path / "recordings"), "daily") + + with db.conn() as c: + paths = [r["path"] for r in + c.execute("SELECT path FROM clip_index ORDER BY path")] + assert str(present) in paths # the real file kept + assert "/recordings/2026-06-03/2026_0603_090000_0002F.MP4" not in paths diff --git a/tests/test_settings_unsubscribe.py b/tests/test_settings_unsubscribe.py new file mode 100644 index 0000000..7cf55a7 --- /dev/null +++ b/tests/test_settings_unsubscribe.py @@ -0,0 +1,62 @@ +"""Settings subscribers must be removable, and the app lifespan must +not leak them across runs. + +The provider is a module-level singleton; each lifespan subscribed +auth/sync/mqtt callbacks and never unsubscribed, so repeated lifespans +(tests, uvicorn reload) accumulated callbacks pinning dead app objects. +""" +from __future__ import annotations + +import bcrypt +from fastapi.testclient import TestClient + +from web import settings as settings_mod + + +def test_subscribe_returns_working_unsubscribe(tmp_config_dir, tmp_recordings_dir): + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + seen = [] + unsub = p.subscribe(lambda keys, snap: seen.append(keys)) + + p.update({"GROUPING": "weekly"}, actor="t") + assert len(seen) == 1 + + unsub() + p.update({"GROUPING": "daily"}, actor="t") + assert len(seen) == 1, "callback still fired after unsubscribe" + + +def test_lifespan_unsubscribes_on_shutdown(tmp_config_dir, tmp_recordings_dir, + monkeypatch): + from web.app import create_app + from web.services.sync_worker import SyncWorker + + class _FakeMqtt: + def __init__(self, **kw): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqtt) + + provider = settings_mod.get_provider() + baseline = len(provider._subscribers) + + for _ in range(3): + app = create_app() + with TestClient(app): + pass # enter + exit one full lifespan + + assert len(provider._subscribers) == baseline, ( + f"subscribers leaked across lifespans: " + f"{len(provider._subscribers)} vs baseline {baseline}" + ) diff --git a/tests/test_switch_pieces.py b/tests/test_switch_pieces.py new file mode 100644 index 0000000..25fab59 --- /dev/null +++ b/tests/test_switch_pieces.py @@ -0,0 +1,47 @@ +"""Tests for the pure timeline-export piece builder.""" +from __future__ import annotations + +from web.services.exporter import build_switch_pieces + + +def _clips(): + return [ + {"path": "/f0.mp4", "channel": "front", "start_ts": 1000, "duration_s": 60}, + {"path": "/f1.mp4", "channel": "front", "start_ts": 1060, "duration_s": 60}, + {"path": "/r0.mp4", "channel": "rear", "start_ts": 1000, "duration_s": 60}, + ] + + +def test_single_segment_within_one_clip(): + segs = [{"channel": "rear", "start_ts": 1010, "end_ts": 1040}] + pieces = build_switch_pieces(segs, _clips()) + assert pieces == [{"path": "/r0.mp4", "ss": 10.0, "t": 30.0}] + + +def test_segment_spans_two_clips(): + segs = [{"channel": "front", "start_ts": 1030, "end_ts": 1090}] + pieces = build_switch_pieces(segs, _clips()) + assert pieces == [ + {"path": "/f0.mp4", "ss": 30.0, "t": 30.0}, + {"path": "/f1.mp4", "ss": 0.0, "t": 30.0}, + ] + + +def test_switch_between_cameras_in_order(): + segs = [ + {"channel": "rear", "start_ts": 1000, "end_ts": 1020}, + {"channel": "front", "start_ts": 1020, "end_ts": 1050}, + ] + pieces = build_switch_pieces(segs, _clips()) + assert pieces == [ + {"path": "/r0.mp4", "ss": 0.0, "t": 20.0}, + {"path": "/f0.mp4", "ss": 20.0, "t": 30.0}, + ] + + +def test_zero_width_and_missing_channel_skipped(): + segs = [ + {"channel": "front", "start_ts": 1000, "end_ts": 1000.02}, + {"channel": "interior", "start_ts": 1000, "end_ts": 1030}, + ] + assert build_switch_pieces(segs, _clips()) == [] diff --git a/tests/test_sync_worker_enospc.py b/tests/test_sync_worker_enospc.py new file mode 100644 index 0000000..1a4b31b --- /dev/null +++ b/tests/test_sync_worker_enospc.py @@ -0,0 +1,136 @@ +"""A full disk must not burn the queue's retry budget. + +ENOSPC was indistinguishable from a flaky socket: each item retried +through its backoff ladder, failed, and the drain moved to the next — +methodically marking the whole queue ``failed`` while the disk stayed +full. ENOSPC now aborts retries, refunds the attempt, raises a sticky +``disk_full`` sync error, and stops the drain for this cycle. +""" +from __future__ import annotations + +import datetime as _dt +import errno +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +import viofosync_lib as vfs +from viofosync_lib import _protocol +from web.db import Database +from web.services import queue as q +from web.services import sync_worker as sw_mod +from web.services.hub import Hub +from web.services.sync_worker import DiskFullError, SyncWorker + +# ---- protocol layer: ENOSPC short-circuits the retry ladder ---- + +class _ENOSPCResponse: + def read(self, n): + raise OSError(errno.ENOSPC, "No space left on device") + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_download_enospc_raises_without_retry(tmp_path, monkeypatch): + monkeypatch.setattr(_protocol, "max_download_attempts", 3) + monkeypatch.setattr(_protocol, "RETRY_BACKOFF", 0) + attempts = {"n": 0} + + def fake_urlopen(url_or_req, *args, **kwargs): + if getattr(url_or_req, "get_method", lambda: "GET")() == "HEAD": + raise OSError("no HEAD") + attempts["n"] += 1 + return _ENOSPCResponse() + + rec = vfs.Recording( + "2026_0101_120000_0001F.MP4", "/DCIM/Movie/x.MP4", 1000, 0, + _dt.datetime(2026, 1, 1, 12, 0), 0, + ) + with patch("urllib.request.urlopen", side_effect=fake_urlopen): + with pytest.raises(OSError) as exc: + _protocol.download_file( + "http://192.0.2.1", rec, str(tmp_path), "", + ) + assert exc.value.errno == errno.ENOSPC + assert attempts["n"] == 1, \ + f"ENOSPC was retried {attempts['n']} times instead of raising" + + +# ---- worker layer: sticky error, attempt refunded, drain stops ---- + +class _Rec: + def __init__(self, filename: str): + self.filename = filename + self.filepath = "/DCIM/Movie" + self.size = 1000 + self.datetime = _dt.datetime(2026, 5, 8, 12, 0, 0) + + +def _snap(): + s = MagicMock() + s.recordings = "/tmp" + s.grouping = "daily" + s.sync_ro_only = False + s.gps_extract = False + s.delete_after_download = False + s.download_attempts = 3 + s.timeout = 5.0 + s.max_attempts = 5 + return s + + +@pytest.fixture +def env(tmp_path: Path): + db = Database(str(tmp_path / "t.db")) + provider = MagicMock() + provider.get.return_value = _snap() + worker = SyncWorker(db, provider, Hub()) + worker._active_address = "192.0.2.1" + q.reconcile(db, [_Rec("A.MP4")], []) + return db, worker + + +async def test_enospc_refunds_attempt_and_sets_sticky_error(env, monkeypatch): + db, worker = env + item = q.next_pending(db) + + def boom(*args, **kwargs): + raise OSError(errno.ENOSPC, "No space left on device") + + monkeypatch.setattr(vfs, "download_file_with", boom) + + with pytest.raises(DiskFullError): + await worker._download_one(item) + + with db.conn() as c: + row = c.execute( + "SELECT state, attempts FROM download_queue WHERE id=?", + (item.id,), + ).fetchone() + assert row["state"] == "pending", "item not returned to pending" + assert row["attempts"] == 0, "full disk burned a retry attempt" + assert worker._last_error_kind == "disk_full" + + +async def test_successful_download_clears_disk_full_error(env, monkeypatch): + db, worker = env + item = q.next_pending(db) + worker._last_error_kind = "disk_full" + + monkeypatch.setattr(vfs, "download_file_with", + lambda *a, **kw: (True, "1 MB/s")) + monkeypatch.setattr(sw_mod, "_refresh_queue_size", + lambda *a, **kw: None) + monkeypatch.setattr(vfs, "get_filepath", + lambda *a, **kw: "/tmp/A.MP4") + + ok = await worker._download_one(item) + + assert ok is True + assert worker._last_error_kind is None, \ + "disk_full error not cleared after space freed" diff --git a/tests/test_sync_worker_error_signals.py b/tests/test_sync_worker_error_signals.py index 17ce50c..57f39f2 100644 --- a/tests/test_sync_worker_error_signals.py +++ b/tests/test_sync_worker_error_signals.py @@ -48,6 +48,27 @@ async def test_check_recordings_writable_clears_previous_error(tmp_path): assert {"type": "sync_error", "kind": None, "message": None} in hub.events +async def test_check_recordings_writable_uses_real_probe_not_os_access( + tmp_path, monkeypatch +): + """Regression: os.access(W_OK) is unreliable on NFS — it checks cached + owner/mode against the local UID and can report a genuinely writable + export as non-writable. The check must probe with a real write instead.""" + snap = types.SimpleNamespace(recordings=str(tmp_path)) + hub = _RecordingHub() + sw = _make_worker(snap, hub) + # Simulate NFS: os.access lies and says "not writable" even though + # an actual create-and-delete in the directory succeeds. + monkeypatch.setattr( + "web.services.sync_worker.os.access", lambda *a, **k: False + ) + ok = await sw._check_recordings_writable() + assert ok is True + assert hub.events == [] + # The probe must leave nothing behind. + assert list(tmp_path.iterdir()) == [] + + async def test_check_recordings_writable_does_not_emit_when_already_clear(tmp_path): snap = types.SimpleNamespace(recordings=str(tmp_path)) hub = _RecordingHub() diff --git a/tests/test_sync_worker_lifecycle.py b/tests/test_sync_worker_lifecycle.py new file mode 100644 index 0000000..f6c31e0 --- /dev/null +++ b/tests/test_sync_worker_lifecycle.py @@ -0,0 +1,106 @@ +"""SyncWorker lifecycle: bounded shutdown + settings-driven start/stop. + +Two regressions pinned here: + +- ``stop()`` awaited the run_coroutine_threadsafe future without any + timeout (the wait_for branch was dead code), so a cycle stuck in an + uncancellable executor call hung SIGTERM shutdown forever. +- Changing ADDRESS / ENABLE_SCHEDULED_SYNC at runtime neither started + nor stopped the worker — only a restart applied them, despite the + settings UI claiming they were applied. +""" +from __future__ import annotations + +import asyncio +import concurrent.futures +from unittest.mock import MagicMock + +import bcrypt +import pytest +from fastapi.testclient import TestClient + +from web import settings as settings_mod +from web.db import Database +from web.services import sync_worker as sw_mod +from web.services.hub import Hub +from web.services.sync_worker import SyncWorker + + +@pytest.fixture +def db(tmp_path) -> Database: + return Database(str(tmp_path / "t.db")) + + +# ---- stop() must not hang on a stuck cycle ---- + +async def test_stop_returns_despite_stuck_cycle(db, monkeypatch): + monkeypatch.setattr(sw_mod, "STOP_TIMEOUT", 0.2, raising=False) + provider = MagicMock() + worker = SyncWorker(db, provider, Hub()) + # Simulate a cycle wedged in an uncancellable executor call: a + # future that never resolves. + worker._task = concurrent.futures.Future() + + done = True + try: + await asyncio.wait_for(worker.stop(), timeout=2.0) + except TimeoutError: + done = False + assert done, "stop() hung on a stuck cycle future" + + +# ---- start/stop decision from settings changes ---- + +def test_sync_worker_action_decision(): + from web.app import _sync_worker_action + + def snap(addr, enabled): + s = MagicMock() + s.address = addr + s.enable_scheduled_sync = enabled + return s + + # Irrelevant keys → no action. + assert _sync_worker_action({"TZ"}, snap("1.2.3.4", True)) is None + # Address set + sync enabled → start. + assert _sync_worker_action({"ADDRESS"}, snap("1.2.3.4", True)) == "start" + # Sync disabled → stop, regardless of address. + assert _sync_worker_action( + {"ENABLE_SCHEDULED_SYNC"}, snap("1.2.3.4", False)) == "stop" + # Address cleared → stop. + assert _sync_worker_action({"ADDRESS"}, snap(None, True)) == "stop" + + +# ---- integration: setting ADDRESS at runtime starts the worker ---- + +def test_setting_address_starts_worker(tmp_config_dir, tmp_recordings_dir, + monkeypatch): + from web.app import create_app + + class _FakeMqtt: + def __init__(self, **kwargs): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + started: list = [] + monkeypatch.setattr(SyncWorker, "start", + lambda self: started.append(True)) + monkeypatch.setattr("web.app.MqttService", _FakeMqtt) + + app = create_app() + with TestClient(app): + assert started == [] # no ADDRESS at boot — worker idle + provider = settings_mod.get_provider() + provider.update({"ADDRESS": "192.0.2.5"}, actor="test") + assert started == [True], \ + "setting ADDRESS at runtime did not start the sync worker" + settings_mod.reset_for_tests() diff --git a/tests/test_task_spawn.py b/tests/test_task_spawn.py new file mode 100644 index 0000000..39d4300 --- /dev/null +++ b/tests/test_task_spawn.py @@ -0,0 +1,51 @@ +"""Detached background tasks must be strongly referenced until done. + +asyncio keeps only a weak reference to a bare create_task result, so +a GC pass can cancel a still-running fire-and-forget task. spawn() +holds a reference until completion and logs (never swallows) a crash. +""" +from __future__ import annotations + +import asyncio +import logging + +from web.services import tasks as task_mod + + +async def test_spawn_tracks_then_releases(): + gate = asyncio.Event() + + async def _work(): + await gate.wait() + + t = task_mod.spawn(_work()) + assert t in task_mod._background, "task not held while running" + gate.set() + await t + assert t not in task_mod._background, "reference not released after done" + + +async def test_spawn_survives_gc_pressure(): + import gc + done = asyncio.Event() + + async def _work(): + await asyncio.sleep(0.05) + done.set() + + task_mod.spawn(_work()) # deliberately keep no local reference + gc.collect() # would collect a bare create_task result + await asyncio.wait_for(done.wait(), timeout=1.0) + + +async def test_spawn_logs_exception(caplog): + async def _boom(): + raise ValueError("detached failure") + + with caplog.at_level(logging.ERROR): + t = task_mod.spawn(_boom()) + await asyncio.gather(t, return_exceptions=True) + + assert any("detached failure" in r.message or "detached failure" in str(r.exc_info) + for r in caplog.records), "detached task exception not logged" + assert t not in task_mod._background diff --git a/tests/test_thumb_atomic.py b/tests/test_thumb_atomic.py new file mode 100644 index 0000000..8bf1c66 --- /dev/null +++ b/tests/test_thumb_atomic.py @@ -0,0 +1,123 @@ +"""Partial ffmpeg output must never become a permanent cache hit. + +thumbs/export-preview wrote ffmpeg output straight to the final cache +path; a killed or failed job left a truncated file that the +``exists() and size > 0`` cache check then served forever. Output must +land at the final path only on success. ensure_thumb also needs a +concurrency cap — a 100-clip day view used to spawn 100 ffmpegs. +""" +from __future__ import annotations + +import asyncio +import glob +import os +import shutil +from pathlib import Path + +from web.services import filmstrip, thumbs + + +class _FakeProc: + def __init__(self, rc: int, hang: bool): + self.returncode = rc + self._hang = hang + + async def wait(self) -> int: + if self._hang: + await asyncio.sleep(60) + return self.returncode + + def kill(self) -> None: + self._hang = False + + +def _fake_ffmpeg(monkeypatch, *, rc: int = 1, hang: bool = False, + write_partial: bool = True, track: dict | None = None): + """Replace subprocess spawning with a fake that (optionally) writes + a partial file at the output path (ffmpeg's last argv) and then + fails or hangs.""" + async def _exec(*argv, **kwargs): + if track is not None: + track["active"] += 1 + track["max"] = max(track["max"], track["active"]) + await asyncio.sleep(0.05) + track["active"] -= 1 + if write_partial: + Path(argv[-1]).write_bytes(b"PARTIAL") + return _FakeProc(rc, hang) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _exec) + monkeypatch.setattr(shutil, "which", lambda name: "/bin/fake-ffmpeg") + + +async def test_failed_thumb_leaves_no_partial_cache(tmp_path, monkeypatch): + _fake_ffmpeg(monkeypatch, rc=1) + rec = tmp_path / "rec" + rec.mkdir() + video = rec / "clip.MP4" + video.write_bytes(b"video") + + got = await thumbs.ensure_thumb(str(rec), 1, str(video)) + + assert got is None + final = thumbs.thumb_path(str(rec), 1) + assert not os.path.exists(final), \ + "failed ffmpeg left a partial thumb that becomes a cache hit" + leftovers = [p for p in glob.glob(final + "*") if not p.endswith(".fail")] + assert leftovers == [], f"temp debris left behind: {leftovers}" + + +async def test_timed_out_thumb_leaves_no_partial_cache(tmp_path, monkeypatch): + _fake_ffmpeg(monkeypatch, rc=0, hang=True) + monkeypatch.setattr(thumbs, "_TIMEOUT_S", 0.1, raising=False) + rec = tmp_path / "rec" + rec.mkdir() + video = rec / "clip.MP4" + video.write_bytes(b"video") + + got = await thumbs.ensure_thumb(str(rec), 1, str(video)) + + assert got is None + assert not os.path.exists(thumbs.thumb_path(str(rec), 1)) + + +async def test_thumb_generation_is_concurrency_capped(tmp_path, monkeypatch): + track = {"active": 0, "max": 0} + _fake_ffmpeg(monkeypatch, rc=1, write_partial=False, track=track) + rec = tmp_path / "rec" + rec.mkdir() + videos = [] + for i in range(10): + v = rec / f"clip{i}.MP4" + v.write_bytes(b"video") + videos.append(v) + + await asyncio.gather(*( + thumbs.ensure_thumb(str(rec), i, str(v)) + for i, v in enumerate(videos) + )) + + assert track["max"] <= 3, \ + f"{track['max']} concurrent ffmpeg thumb jobs (want <= 3)" + + +async def test_failed_sprite_montage_leaves_no_partial(tmp_path, monkeypatch): + sprite = str(tmp_path / "42.jpg") + + async def _fake_run(cmd, timeout): + dest = cmd[-1] + Path(dest).write_bytes(b"X") + # Tile extractions (inside the temp .tiles_ dir) succeed; the + # final montage fails after writing partial output. + return 0 if ".tiles_" in dest else 1 + + monkeypatch.setattr(filmstrip, "_run_ffmpeg", _fake_run) + + ok = await filmstrip.generate_sprite_at( + "/bin/fake-ffmpeg", str(tmp_path / "in.mp4"), sprite, [0.5, 1.5], + ) + + assert ok is False + assert not os.path.exists(sprite), \ + "failed montage left a partial sprite at the cache path" + assert glob.glob(sprite + "*") == [], "partial temp sprite left behind" diff --git a/tests/test_thumb_failcache.py b/tests/test_thumb_failcache.py new file mode 100644 index 0000000..2c0f0c3 --- /dev/null +++ b/tests/test_thumb_failcache.py @@ -0,0 +1,79 @@ +"""Thumbnail sweep must not re-attempt clips that can't produce a thumb. + +Regression: ``ensure_thumb`` returned None on ffmpeg failure and left no +marker, so un-thumbable clips (short/corrupt/partial) were re-selected and +re-run through ffmpeg on every sweep. With a sweep after every working +cycle (and on pause) that was a recurring CPU storm. +""" +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest + +from web.db import Database +from web.services import scanner, thumbs + + +@pytest.fixture +def db(tmp_path: Path) -> Database: + return Database(str(tmp_path / ".viofosync.db")) + + +def _add_clip(db: Database, path: str, clip_id: int = 1) -> int: + with db.write() as c: + c.execute( + "INSERT INTO clip_index (id, path, basename, timestamp, camera, " + "sequence, scanned_at) VALUES (?,?,?,?,?,?,?)", + (clip_id, path, os.path.basename(path), 0, "F", 0, 0), + ) + return clip_id + + +def test_mark_failed_then_skipped(tmp_path: Path): + rec = tmp_path / "rec" + rec.mkdir() + video = rec / "clip.MP4" + video.write_bytes(b"not a real video") + # A fresh failure marker (recorded after the video was written) means + # "don't bother trying again until the file changes". + thumbs.mark_failed(str(rec), 1) + assert thumbs.failed_recently(str(rec), 1, str(video)) is True + + +def test_stale_marker_retried_after_file_changes(tmp_path: Path): + rec = tmp_path / "rec" + rec.mkdir() + video = rec / "clip.MP4" + video.write_bytes(b"old") + thumbs.mark_failed(str(rec), 1) + # The clip is later rewritten (e.g. a partial import got redone) — its + # mtime moves past the marker, so the thumb is worth another attempt. + time.sleep(0.01) + os.utime(str(video), None) + assert thumbs.failed_recently(str(rec), 1, str(video)) is False + + +async def test_sweep_skips_failed_clip_on_next_pass(tmp_path: Path, db: Database): + rec = tmp_path / "rec" + rec.mkdir() + video = rec / "clip.MP4" + video.write_bytes(b"not a real video") + _add_clip(db, str(video)) + + calls = {"n": 0} + + async def _fake_ensure(recordings, clip_id, path): + calls["n"] += 1 + thumbs.mark_failed(recordings, clip_id) # simulate ffmpeg failure + return None + + import unittest.mock as _m + with _m.patch.object(thumbs, "ensure_thumb", _fake_ensure): + await scanner.sweep_missing_thumbs(db, str(rec)) + await scanner.sweep_missing_thumbs(db, str(rec)) + + # First sweep attempts it once; the second must skip it. + assert calls["n"] == 1 diff --git a/tests/test_thumbs.py b/tests/test_thumbs.py new file mode 100644 index 0000000..4486662 --- /dev/null +++ b/tests/test_thumbs.py @@ -0,0 +1,46 @@ +"""Tests for the thumbnail service (ffmpeg mocked).""" +from __future__ import annotations + +from pathlib import Path + +from web.services import thumbs + + +class _HangProc: + """Fake ffmpeg child: kill() records, wait() counts body runs.""" + returncode = None + + def __init__(self): + self.killed = False + self.reaped = 0 + + def kill(self): + self.killed = True + + async def wait(self): + self.reaped += 1 + return 0 + + +async def _raise_timeout(coro, timeout): + # Close the inner proc.wait() coroutine so it isn't left un-awaited + # (the suite runs under filterwarnings=error), then simulate a timeout. + coro.close() + raise TimeoutError + + +async def test_ensure_thumb_reaps_child_on_timeout(tmp_path: Path, monkeypatch): + rec = str(tmp_path) + monkeypatch.setattr(thumbs.shutil, "which", lambda _n: "/usr/bin/ffmpeg") + fake = _HangProc() + + async def fake_exec(*a, **k): + return fake + + monkeypatch.setattr(thumbs.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(thumbs.asyncio, "wait_for", _raise_timeout) + + result = await thumbs.ensure_thumb(rec, 7, "/x.mp4") + assert result is None + assert fake.killed is True + assert fake.reaped == 1 # proc.wait() awaited after kill -> child reaped diff --git a/tests/test_timeline_endpoint.py b/tests/test_timeline_endpoint.py new file mode 100644 index 0000000..fa4ec8d --- /dev/null +++ b/tests/test_timeline_endpoint.py @@ -0,0 +1,196 @@ +"""Tests for build_route_payload + GET /api/archive/timeline.""" +from __future__ import annotations + +import logging +from pathlib import Path + +import pytest + +from web.db import Database +from web.routers import archive + + +def test_build_route_payload_empty_day(tmp_path: Path): + """No gpx clips for the date -> empty journeys/stops, point_count 0.""" + db = Database(str(tmp_path / "t.db")) + payload = archive.build_route_payload(db, str(tmp_path), "2026-06-02", None) + assert payload["date"] == "2026-06-02" + assert payload["point_count"] == 0 + assert payload["journeys"] == [] + assert payload["stops"] == [] + + +class _FakeMqttService: + def __init__(self, **kwargs): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): + return {"state": "idle", "detail": None, "last_published_at": None} + + +@pytest.fixture +def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) + + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + yield c + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def _insert_clip(app, clip_id, ts, camera, duration_s, date="2026-06-02"): + with app.state.db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(id, path, basename, group_name, timestamp, camera, " + " sequence, event_type, has_gpx, gps_examined, scanned_at, duration_s) " + "VALUES (?,?,?,?,?,?,?,?,0,0,?,?)", + (clip_id, f"/rec/{clip_id}.MP4", f"{clip_id}.MP4", date, + ts, camera, clip_id, "normal", ts, duration_s), + ) + + +def test_timeline_bad_date_400(logged_in_client): + r = logged_in_client.get("/api/archive/timeline?date=nonsense") + assert r.status_code == 400 + + +def test_timeline_day_mode_channels_clips_bounds(logged_in_client): + app = logged_in_client.app + _insert_clip(app, 1, 1_717_312_440, "F", 60.0) + _insert_clip(app, 2, 1_717_312_440, "R", 60.0) + _insert_clip(app, 3, 1_717_312_500, "F", 60.0) + + r = logged_in_client.get("/api/archive/timeline?date=2026-06-02") + assert r.status_code == 200 + body = r.json() + assert [ch["key"] for ch in body["channels"]] == ["front", "rear"] + assert body["channels"][0]["label"] == "Front" + assert len(body["clips"]) == 3 + assert body["bounds"]["start_ts"] == 1_717_312_440 + assert body["bounds"]["end_ts"] == 1_717_312_560 + assert body["gps"] is None + + +def test_timeline_journey_mode_windows_clips(logged_in_client, monkeypatch): + app = logged_in_client.app + _insert_clip(app, 1, 1_717_312_440, "F", 60.0) + _insert_clip(app, 2, 1_717_312_500, "F", 60.0) + _insert_clip(app, 3, 1_717_313_040, "F", 60.0) + + fake_route = { + "date": "2026-06-02", + "point_count": 5, + "journeys": [{"start_ts": 1_717_312_440, "end_ts": 1_717_312_560}], + "stops": [], + } + monkeypatch.setattr( + "web.routers.archive.build_route_payload", + lambda db, recordings, date, geocoder: fake_route, + ) + + r = logged_in_client.get("/api/archive/timeline?date=2026-06-02&journey=0") + assert r.status_code == 200 + body = r.json() + ids = sorted(c["id"] for c in body["clips"]) + assert ids == [1, 2] + assert [ch["key"] for ch in body["channels"]] == ["front"] + assert body["bounds"]["start_ts"] == 1_717_312_440 + assert body["bounds"]["end_ts"] == 1_717_312_560 + assert body["gps"] is not None + + +def test_timeline_journey_out_of_range_404(logged_in_client, monkeypatch): + app = logged_in_client.app + _insert_clip(app, 1, 1_717_312_440, "F", 60.0) + monkeypatch.setattr( + "web.routers.archive.build_route_payload", + lambda db, recordings, date, geocoder: { + "date": "2026-06-02", "point_count": 0, "journeys": [], "stops": [], + }, + ) + r = logged_in_client.get("/api/archive/timeline?date=2026-06-02&journey=0") + assert r.status_code == 404 + + +def test_timeline_open_logs_clip_count(logged_in_client, caplog): + """Opening the editor logs how many clips (= filmstrip jobs) it will + drive, so a NAS CPU spike is traceable from the Logs tab.""" + app = logged_in_client.app + _insert_clip(app, 1, 1_717_312_440, "F", 60.0) + _insert_clip(app, 2, 1_717_312_440, "R", 60.0) + + with caplog.at_level(logging.INFO, logger="viofosync.archive"): + r = logged_in_client.get("/api/archive/timeline?date=2026-06-02") + assert r.status_code == 200 + + msgs = [r.getMessage() for r in caplog.records] + assert any("timeline: date=2026-06-02" in m and "2 clip(s)" in m for m in msgs) + + +# --- fallback durations: the editor needs a non-zero duration per clip to +# render blocks and resolve footage at the playhead. Until ffprobe has filled +# duration_s, derive it from the gap to the next clip on the same channel so +# the editor works immediately instead of showing empty tracks. + + +def test_timeline_fills_missing_duration_from_gap(logged_in_client): + app = logged_in_client.app + base = 1_717_312_440 + _insert_clip(app, 1, base, "F", None) + _insert_clip(app, 2, base + 45, "F", None) + body = logged_in_client.get("/api/archive/timeline?date=2026-06-02").json() + clips = sorted(body["clips"], key=lambda c: c["start_ts"]) + assert clips[0]["duration_s"] == 45 # gap to next clip + assert clips[1]["duration_s"] == archive.FALLBACK_DEFAULT_S # last -> default + + +def test_timeline_caps_fallback_for_large_gap(logged_in_client): + app = logged_in_client.app + base = 1_717_312_440 + _insert_clip(app, 1, base, "F", None) + _insert_clip(app, 2, base + 99_999, "F", None) # parking-sized gap + body = logged_in_client.get("/api/archive/timeline?date=2026-06-02").json() + clips = sorted(body["clips"], key=lambda c: c["start_ts"]) + assert clips[0]["duration_s"] == archive.FALLBACK_MAX_S # capped + + +def test_timeline_gap_is_per_channel(logged_in_client): + app = logged_in_client.app + base = 1_717_312_440 + _insert_clip(app, 1, base, "F", None) + _insert_clip(app, 2, base + 10, "R", None) # other channel, ignored + _insert_clip(app, 3, base + 60, "F", None) + body = logged_in_client.get("/api/archive/timeline?date=2026-06-02").json() + fronts = sorted( + (c for c in body["clips"] if c["channel"] == "front"), + key=lambda c: c["start_ts"], + ) + assert fronts[0]["duration_s"] == 60 # gap to next FRONT, not the rear + + +def test_timeline_keeps_real_duration(logged_in_client): + app = logged_in_client.app + _insert_clip(app, 1, 1_717_312_440, "F", 42.0) + body = logged_in_client.get("/api/archive/timeline?date=2026-06-02").json() + assert body["clips"][0]["duration_s"] == 42.0 diff --git a/tests/test_timeline_export.py b/tests/test_timeline_export.py new file mode 100644 index 0000000..256bf3f --- /dev/null +++ b/tests/test_timeline_export.py @@ -0,0 +1,262 @@ +"""Tests for the timeline export job (enqueue + render, ffmpeg mocked).""" +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from web.db import Database +from web.services.exporter import ExportWorker + + +@pytest.fixture +def db(tmp_path: Path) -> Database: + return Database(str(tmp_path / "t.db")) + + +async def _noop(_e): # broadcast stub + pass + + +def _worker(db): + return ExportWorker(db=db, provider=MagicMock(), broadcast=_noop) + + +def test_enqueue_timeline_stores_plan_and_range(db, monkeypatch): + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + segs = [ + {"channel": "rear", "start_ts": 1000.0, "end_ts": 1020.0}, + {"channel": "front", "start_ts": 1020.0, "end_ts": 1050.0}, + ] + job_id = _worker(db).enqueue_timeline(segs, encoder="software") + with db.conn() as c: + row = c.execute( + "SELECT type, clip_ids, clip_start, clip_end FROM export_jobs WHERE id=?", + (job_id,), + ).fetchone() + import json + assert row["type"] == "timeline" + payload = json.loads(row["clip_ids"]) + assert payload["encoder"] == "software" + assert len(payload["segments"]) == 2 + assert row["clip_start"] == 1000 + assert row["clip_end"] == 1050 + + +def test_enqueue_timeline_rejects_empty(db, monkeypatch): + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + with pytest.raises(ValueError): + _worker(db).enqueue_timeline([], encoder="software") + + +def test_enqueue_timeline_rejects_bad_window(db, monkeypatch): + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + with pytest.raises(ValueError): + _worker(db).enqueue_timeline( + [{"channel": "front", "start_ts": 50.0, "end_ts": 50.0}], + encoder="software", + ) + + +def _insert_clip(db, clip_id, ts, camera, dur, path): + with db.write() as c: + c.execute( + "INSERT INTO clip_index " + "(id, path, basename, group_name, timestamp, camera, " + " sequence, event_type, has_gpx, gps_examined, scanned_at, duration_s) " + "VALUES (?,?,?,?,?,?,?,?,0,0,?,?)", + (clip_id, path, f"{clip_id}.MP4", "2026-06-02", + ts, camera, clip_id, "normal", ts, dur), + ) + + +async def test_run_timeline_video_only_with_continuous_front_audio( + db, tmp_path, monkeypatch, +): + """Timeline video is cut per-segment (picture only); audio is one + continuous front-camera track muxed at the end, never re-cut at switches.""" + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + _insert_clip(db, 1, 1000, "F", 60.0, "/rec/f0.mp4") + _insert_clip(db, 2, 1000, "R", 60.0, "/rec/r0.mp4") + snap = MagicMock() + snap.recordings = str(tmp_path) + provider = MagicMock() + provider.get.return_value = snap + worker = ExportWorker(db=db, provider=provider, broadcast=_noop) + + calls = [] + + async def fake_run_ffmpeg(job_id, args, total, **kw): + calls.append(list(args)) + Path(args[-1]).write_bytes(b"\0") + return 0, "" + + async def fake_probe_res(path): + return (1920, 1080) + + monkeypatch.setattr(worker, "_run_ffmpeg", fake_run_ffmpeg) + monkeypatch.setattr(worker, "_probe_resolution", fake_probe_res) + finishes = [] + monkeypatch.setattr(worker, "_finish", + lambda jid, ok, err, out: finishes.append((ok, err, out))) + + segs = [ + {"channel": "rear", "start_ts": 1000, "end_ts": 1020}, + {"channel": "front", "start_ts": 1020, "end_ts": 1050}, + ] + import json as _json + job = {"id": 5, "type": "timeline", + "clip_ids": _json.dumps({"segments": segs, "encoder": "software"})} + await worker._run_job(job) + + assert finishes and finishes[-1][0] is True, finishes + + # Video segments are encoded picture-only (-an) and carry no audio codec. + video = [a for a in calls if "-an" in a] + assert len(video) == 2 + assert all("-c:a" not in a for a in video) + # Rear window first, sourced from the rear file. + assert "/rec/r0.mp4" in video[0] + assert video[0][video[0].index("-ss") + 1] == "0.0" + assert "scale=1920:1080,setsar=1" in video[0][video[0].index("-vf") + 1] + # Front window second, sourced from the front file. + assert "/rec/f0.mp4" in video[1] + assert video[1][video[1].index("-ss") + 1] == "20.0" + + # Audio is a single continuous front-camera track spanning the WHOLE + # export — including the rear video window — so it is sourced from the + # front file and never from the rear file. + audio = [a for a in calls if "-vn" in a] + assert len(audio) == 1 + assert "/rec/f0.mp4" in audio[0] + assert audio[0][audio[0].index("-ss") + 1] == "0.0" + assert audio[0][audio[0].index("-t") + 1] == "50.0" + assert all("/rec/r0.mp4" not in a for a in audio) + + # Final mux pads audio to the video length and copies the picture. + mux = next(a for a in calls if "[1:a]apad[aud]" in a) + assert "0:v:0" in mux + assert "[aud]" in mux + assert "-shortest" in mux + + +async def test_run_timeline_no_front_footage_yields_silent_video( + db, tmp_path, monkeypatch, +): + """If no front footage exists in the span there is no audio source, so + the export succeeds as a silent timeline video (no audio encode, no mux).""" + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + _insert_clip(db, 1, 1000, "R", 60.0, "/rec/r0.mp4") + snap = MagicMock() + snap.recordings = str(tmp_path) + provider = MagicMock() + provider.get.return_value = snap + worker = ExportWorker(db=db, provider=provider, broadcast=_noop) + + calls = [] + + async def fake_run_ffmpeg(job_id, args, total, **kw): + calls.append(list(args)) + Path(args[-1]).write_bytes(b"\0") + return 0, "" + + async def fake_probe_res(path): + return (1920, 1080) + + monkeypatch.setattr(worker, "_run_ffmpeg", fake_run_ffmpeg) + monkeypatch.setattr(worker, "_probe_resolution", fake_probe_res) + finishes = [] + monkeypatch.setattr(worker, "_finish", + lambda jid, ok, err, out: finishes.append((ok, err, out))) + + segs = [{"channel": "rear", "start_ts": 1000, "end_ts": 1020}] + import json as _json + job = {"id": 7, "type": "timeline", + "clip_ids": _json.dumps({"segments": segs, "encoder": "software"})} + await worker._run_job(job) + + assert finishes and finishes[-1][0] is True, finishes + assert not any("-vn" in a for a in calls) # no audio encode + assert not any("apad" in tok for a in calls for tok in a) # no mux + + +async def test_run_timeline_no_footage_fails(db, tmp_path, monkeypatch): + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + snap = MagicMock() + snap.recordings = str(tmp_path) + provider = MagicMock() + provider.get.return_value = snap + worker = ExportWorker(db=db, provider=provider, broadcast=_noop) + finishes = [] + monkeypatch.setattr(worker, "_finish", + lambda jid, ok, err, out: finishes.append((ok, err))) + import json as _json + job = {"id": 6, "type": "timeline", + "clip_ids": _json.dumps( + {"segments": [{"channel": "front", "start_ts": 1, "end_ts": 9}], + "encoder": "software"})} + await worker._run_job(job) + assert finishes[-1][0] is False + + +class _FakeMqttService: + def __init__(self, **k): pass + def start(self): pass + async def stop(self): pass + async def on_settings_changed(self, keys, snap): pass + def get_status(self): + return {"state": "idle", "detail": None, "last_published_at": None} + + +@pytest.fixture +def logged_in_client(tmp_config_dir, tmp_recordings_dir, monkeypatch): + import bcrypt + from fastapi.testclient import TestClient + + from web import settings as settings_mod + from web.app import create_app + from web.services.sync_worker import SyncWorker + + digest = bcrypt.hashpw(b"pw" * 8, bcrypt.gensalt()).decode() + settings_mod.reset_for_tests() + p = settings_mod.get_provider() + data = p._store.load() + data["WEB_PASSWORD_HASH"] = digest + p._store.write(data) + settings_mod.reset_for_tests() + monkeypatch.setattr(SyncWorker, "start", lambda self: None) + monkeypatch.setattr("web.app.MqttService", _FakeMqttService) + app = create_app() + c = TestClient(app) + c.__enter__() + c.post("/api/auth/login", json={"password": "pwpwpwpwpwpwpwpw"}) + yield c + c.__exit__(None, None, None) + settings_mod.reset_for_tests() + + +def test_post_timeline_export_creates_job(logged_in_client, monkeypatch): + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + logged_in_client.app.state.export_encoders = {"software": True} + csrf = logged_in_client.get("/api/auth/csrf").json()["csrf"] + r = logged_in_client.post("/api/exports", json={ + "type": "timeline", + "segments": [ + {"channel": "rear", "start_ts": 1000.0, "end_ts": 1020.0}, + {"channel": "front", "start_ts": 1020.0, "end_ts": 1050.0}, + ], + "encoder": "software", + }, headers={"x-csrf-token": csrf}) + assert r.status_code == 200, r.text + assert "job_id" in r.json() + + +def test_post_timeline_requires_segments(logged_in_client, monkeypatch): + monkeypatch.setattr("web.services.exporter.ffmpeg_available", lambda: True) + logged_in_client.app.state.export_encoders = {"software": True} + csrf = logged_in_client.get("/api/auth/csrf").json()["csrf"] + r = logged_in_client.post("/api/exports", json={ + "type": "timeline", "clip_ids": [], "encoder": "software"}, + headers={"x-csrf-token": csrf}) + assert r.status_code in (400, 422) diff --git a/tests/test_upload_offloop.py b/tests/test_upload_offloop.py new file mode 100644 index 0000000..8c14365 --- /dev/null +++ b/tests/test_upload_offloop.py @@ -0,0 +1,89 @@ +"""The upload route must not block the event loop. + +Its quota check (make_room_for) walks the whole archive in quota +mode, and the chunk writes hit a (typically NAS) volume — both used +to run synchronously inside the async handler, serialising the whole +server behind disk latency. +""" +from __future__ import annotations + +import asyncio +import time +from types import SimpleNamespace +from unittest.mock import MagicMock + +from starlette.requests import Request + +from web.db import Database +from web.routers import imports as imports_router + + +def _fake_app(tmp_path) -> SimpleNamespace: + snap = MagicMock() + snap.recordings = str(tmp_path / "rec") + snap.grouping = "daily" + snap.import_path = None + snap.retention_disk_pct = 0 + snap.recordings_quota_gb = 0 + snap.retention_protect_ro = True + provider = MagicMock() + provider.get.return_value = snap + db = Database(str(tmp_path / "t.db")) + return SimpleNamespace( + state=SimpleNamespace(settings_provider=provider, db=db) + ) + + +def _request(app, name: str, body: bytes) -> Request: + messages = [{"type": "http.request", "body": body, "more_body": False}] + + async def receive(): + return messages.pop(0) + + scope = { + "type": "http", + "method": "POST", + "path": "/api/import/upload", + "query_string": b"", + "headers": [ + (b"x-import-path", name.encode()), + (b"x-import-size", str(len(body)).encode()), + ], + "app": app, + } + return Request(scope, receive) + + +async def test_upload_does_not_block_loop(tmp_path, monkeypatch): + app = _fake_app(tmp_path) + + def _slow_make_room(*args, **kwargs): + time.sleep(0.3) # simulate the quota-mode archive walk + return True + + monkeypatch.setattr( + imports_router._retention, "make_room_for", _slow_make_room + ) + monkeypatch.setattr( + imports_router._retention, "import_exclude_set", + lambda *a, **k: set(), + ) + + ticks = 0 + + async def _ticker(): + nonlocal ticks + while True: + await asyncio.sleep(0.02) + ticks += 1 + + t = asyncio.create_task(_ticker()) + try: + res = await imports_router.upload( + _request(app, "2026_0101_120000_0001F.MP4", b"x" * 1024) + ) + finally: + t.cancel() + + assert res["status"] == "imported" + assert ticks >= 5, f"event loop starved during upload ({ticks} ticks)" diff --git a/viofosync_lib/__init__.py b/viofosync_lib/__init__.py index 8e24311..c30c58c 100644 --- a/viofosync_lib/__init__.py +++ b/viofosync_lib/__init__.py @@ -42,20 +42,17 @@ def download_file_with( socket_timeout: float | None = None, **kwargs, ): - """Call :func:`download_file` with a temporarily-overridden - ``max_attempts`` / ``socket_timeout``, restoring the prior - values on exit. Avoids direct mutation of module globals from - callers.""" + """Call :func:`download_file` with per-call ``max_attempts`` / + ``socket_timeout`` overrides. Passes them straight through as + parameters (download_file resolves None to the module defaults), + so two concurrent downloads never clobber each other's settings.""" from . import _protocol as _proto - saved = (_proto.max_download_attempts, _proto.socket_timeout) - if max_attempts is not None: - _proto.max_download_attempts = max_attempts - if socket_timeout is not None: - _proto.socket_timeout = socket_timeout - try: - return _proto.download_file(*args, **kwargs) - finally: - _proto.max_download_attempts, _proto.socket_timeout = saved + return _proto.download_file( + *args, + max_attempts=max_attempts, + socket_timeout=socket_timeout, + **kwargs, + ) def delete_dashcam_file( diff --git a/viofosync_lib/_gpx.py b/viofosync_lib/_gpx.py index e4b375a..5f04106 100644 --- a/viofosync_lib/_gpx.py +++ b/viofosync_lib/_gpx.py @@ -309,6 +309,16 @@ def parse_moov(in_fh): sub_atom_size, sub_atom_type = get_atom_info( in_fh.read(8) ) + # An atom can't be smaller than its own 8-byte + # header. A zero/short size here (truncated + # power-loss clip) used to loop forever re-reading + # the same offset. + if sub_atom_size < 8: + logger.warning( + "corrupt atom (size %d) at offset %d; " + "stopping GPS walk", sub_atom_size, sub_offset + ) + break if sub_atom_type == 'gps ': gps_offset = 16 + sub_offset diff --git a/viofosync_lib/_protocol.py b/viofosync_lib/_protocol.py index 82e3e19..6b2c3ce 100644 --- a/viofosync_lib/_protocol.py +++ b/viofosync_lib/_protocol.py @@ -4,13 +4,16 @@ chunked atomic byte downloader. The module-level ``socket_timeout`` and ``max_download_attempts`` -globals are intentionally exposed at module scope: the wrapper -:func:`viofosync_lib.download_file_with` mutates them around a -single download call to apply per-request overrides. +globals are the defaults used when a caller doesn't pass per-call +overrides; :func:`download_file` accepts ``socket_timeout`` / +``max_attempts`` keyword args (threaded through by +:func:`viofosync_lib.download_file_with`) so concurrent downloads +never share mutable state. """ from __future__ import annotations import datetime +import errno import http.client import logging import os @@ -39,7 +42,7 @@ class DownloadCancelled(Exception): """ -# Tunables (mutated by viofosync_lib.download_file_with). +# Defaults used when download_file gets no per-call override. socket_timeout = 10.0 DEFAULT_DOWNLOAD_ATTEMPTS = 1 max_download_attempts = DEFAULT_DOWNLOAD_ATTEMPTS @@ -51,6 +54,21 @@ def parse_viofo_datetime(time_str): return datetime.datetime.strptime(time_str, "%Y/%m/%d %H:%M:%S") +def _interruptible_sleep(seconds, cancel_check): + """Sleep up to ``seconds``, polling ``cancel_check`` so a + pause/stop/unreachable signal is honoured within ~100ms instead + of after the full backoff. Raises DownloadCancelled if asked to + stop.""" + deadline = time.monotonic() + seconds + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + if cancel_check is not None and cancel_check(): + raise DownloadCancelled("cancelled during retry backoff") + time.sleep(min(0.1, remaining)) + + def get_dashcam_filenames( base_url, *, @@ -66,7 +84,12 @@ def get_dashcam_filenames( try: url = f"{base_url}/?custom=1&cmd=3015&par=1" request = urllib.request.Request(url) - response = urllib.request.urlopen(request) + # Read the tunable at call time — download_file_with + # temporarily overrides it. Without a timeout a half-open + # connection wedges the sync worker's thread forever. + response = urllib.request.urlopen( + request, timeout=socket_timeout + ) if response.getcode() != 200: raise RuntimeError( @@ -257,19 +280,28 @@ def human_speed(num_bytes, elapsed): def download_file(base_url, recording, destination, group_name, - progress_sink=None, cancel_check=None): + progress_sink=None, cancel_check=None, + *, max_attempts=None, socket_timeout=None): """Downloads a file from the Viofo dashcam to the destination. Returns (downloaded: bool, speed_str: str|None). - Uses HEAD to check size, retries up to max_download_attempts, - and verifies integrity after download. + Uses HEAD to check size, retries up to ``max_attempts``, and + verifies integrity after download. Optional args (used by the web UI): progress_sink: object with item_started/item_progress/ item_finished methods; see viofosync_lib.ProgressSink. cancel_check: callable returning True if the download should be aborted (e.g. reachability lost, user stopped). + max_attempts / socket_timeout: per-call overrides. Passed as + parameters (not via module globals) so concurrent downloads + can't clobber each other; default to the module-level values + when omitted. """ + # Resolve per-call overrides against the module defaults. ``timeout`` + # shadows the module global of the same name within this function. + attempts = max_attempts if max_attempts is not None else max_download_attempts + timeout = socket_timeout if socket_timeout is not None else globals()["socket_timeout"] sink = progress_sink if group_name: group_filepath = os.path.join(destination, group_name) @@ -295,11 +327,16 @@ def download_file(base_url, recording, destination, group_name, ) url = f"{base_url}/{cleaned.lstrip('/')}" - # Check expected size via HEAD. + # Check expected size via HEAD. Some firmwares drop HEAD under + # load — fall back to the listing size so integrity verification + # still runs (a connection closed cleanly mid-stream otherwise + # archives a truncated file as a success). try: - expected_size = get_remote_size(url, socket_timeout) + expected_size = get_remote_size(url, timeout) except Exception: expected_size = None + if expected_size is None: + expected_size = recording.size # Skip if already downloaded and size matches. if os.path.exists(dest_filepath): @@ -350,13 +387,13 @@ def download_file(base_url, recording, destination, group_name, sink.item_started(recording.filename, expected_size) try: - for attempt in range(1, max_download_attempts + 1): + for attempt in range(1, attempts + 1): try: start = time.perf_counter() bytes_done = 0 last_emit = start with urllib.request.urlopen( - url, timeout=socket_timeout + url, timeout=timeout ) as resp, open(tmp_path, "wb") as out: while True: if cancel_check is not None and cancel_check(): @@ -391,12 +428,17 @@ def download_file(base_url, recording, destination, group_name, ) raise except Exception as e: + if isinstance(e, OSError) and e.errno == errno.ENOSPC: + # Full disk: retrying can only fail the same way + # and would burn the item's retry budget. Raise so + # the caller can surface a sticky disk error. + raise logger.warning( f"Download attempt {attempt} failed for " f"{recording.filename}: {e}" ) - if attempt < max_download_attempts: - time.sleep(RETRY_BACKOFF * attempt) + if attempt < attempts: + _interruptible_sleep(RETRY_BACKOFF * attempt, cancel_check) continue actual_size = os.path.getsize(tmp_path) @@ -410,8 +452,8 @@ def download_file(base_url, recording, destination, group_name, f"{human_size(actual_size)}/" f"{human_size(expected_size)}" ) - if attempt < max_download_attempts: - time.sleep(RETRY_BACKOFF * attempt) + if attempt < attempts: + _interruptible_sleep(RETRY_BACKOFF * attempt, cancel_check) continue # Success — atomic move into place. @@ -431,7 +473,7 @@ def download_file(base_url, recording, destination, group_name, # All attempts exhausted. logger.error( f"Failed to download {recording.filename} " - f"after {max_download_attempts} attempts" + f"after {attempts} attempts" ) if sink is not None: sink.item_finished( diff --git a/web/app.py b/web/app.py index b0c933e..318eddd 100644 --- a/web/app.py +++ b/web/app.py @@ -34,8 +34,10 @@ from .routers import storage as storage_router from .routers import imports as imports_router from .routers import logs as logs_router +from .services import durations as _dur_mod from .services import retention as _ret_mod from .services import scanner +from .services import tasks as _tasks from .services.exporter import ( ExportWorker, ffmpeg_available, @@ -54,6 +56,20 @@ STATIC_DIR = os.path.join(os.path.dirname(__file__), "static") +def _sync_worker_action(keys: set, snap) -> str | None: + """Decide how a settings change affects the sync worker. + + Returns ``"start"``, ``"stop"``, or None (no relevant change). + Pure so the decision is unit-testable apart from the lifespan + wiring. + """ + if not ({"ADDRESS", "ENABLE_SCHEDULED_SYNC"} & keys): + return None + if snap.enable_scheduled_sync and snap.address: + return "start" + return "stop" + + @asynccontextmanager async def lifespan(app: FastAPI): """Startup / shutdown hook. @@ -81,7 +97,10 @@ def _on_settings_changed( if "SESSION_SECRET" in keys: app.state.auth.rotate_secret(snap.session_secret) - provider.subscribe(_on_settings_changed) + # Collect unsubscribe handles so shutdown can detach every + # callback — the provider is a module-level singleton, so leaking + # callbacks across lifespans pins dead app objects. + app.state.settings_unsubscribes = [provider.subscribe(_on_settings_changed)] db_path = default_db_path() migrate_legacy_db_path(db_path) @@ -98,6 +117,12 @@ def _on_settings_changed( log.info("reset %d orphan download row(s) to pending", n_dl) if n_jobs: log.info("marked %d orphan export job(s) as failed", n_jobs) + # Drop partial/unreferenced files a crashed render left in + # .exports (they'd otherwise count against the recordings quota). + try: + _exp_mod.sweep_orphan_exports(app.state.db, s.recordings) + except Exception: # pragma: no cover — non-fatal + log.exception("export orphan sweep failed") log.info( "viofosync web UI ready on http://%s:%d", s.host, s.port @@ -108,6 +133,18 @@ def _on_settings_changed( ) async def _background_scan() -> None: + # Salvage staged clips from a crash mid-import before the + # scan, so the recovered files get indexed in the same pass. + try: + from .services import importer as _imp_mod + rec = await asyncio.to_thread( + _imp_mod.recover_staging, app.state.db, s, + ) + if rec["recovered"]: + log.info("recovered %d staged clip(s) from a previous " + "interrupted import", rec["recovered"]) + except Exception as e: # pragma: no cover — non-fatal + log.warning("staging recovery failed: %s", e) try: log.info("initial archive scan: starting (%s)", s.recordings) loop = asyncio.get_running_loop() @@ -128,6 +165,10 @@ async def _background_scan() -> None: ) except Exception as e: # pragma: no cover — non-fatal log.warning("thumb sweep failed: %s", e) + try: + await _dur_mod.sweep_missing_durations(app.state.db) + except Exception as e: # pragma: no cover — non-fatal + log.warning("duration sweep failed: %s", e) app.state.initial_scan_task = asyncio.create_task(_background_scan()) @@ -140,6 +181,7 @@ async def _background_retention() -> None: disk_pct=s.retention_disk_pct, protect_ro=s.retention_protect_ro, quota_gb=s.recordings_quota_gb, + protect_ids=_exp_mod.export_protect_ids(app.state.db), ) except Exception: # pragma: no cover — non-fatal log.exception("startup retention sweep failed") @@ -153,6 +195,9 @@ async def _background_retention() -> None: settings_provider=provider, session=app.state.download_session, ) + # Threadpool route handlers emit events without a loop reference; + # the hub falls back to this bound loop. + app.state.hub.bind_loop(asyncio.get_running_loop()) # Now that db + hub + loop exist, let the log handler persist and # live-broadcast records. Records logged earlier in startup were # buffered by the handler and flush when the drain task starts. @@ -206,6 +251,23 @@ async def _background_retention() -> None: "ADDRESS not set — sync worker idle until configured" ) + def _on_sync_settings_changed(keys, snap) -> None: + # Apply ADDRESS / ENABLE_SCHEDULED_SYNC at runtime — these + # are not restart-required keys, so the settings UI reports + # them as applied; previously they did nothing until reboot. + worker = app.state.sync_worker + action = _sync_worker_action(keys, snap) + if action == "start" and not worker._is_running(): + log.info("settings change: starting sync worker") + worker.start() + elif action == "stop" and worker._is_running(): + log.info("settings change: stopping sync worker") + _tasks.spawn(worker.stop(), name="sync-worker-stop") + + app.state.settings_unsubscribes.append( + provider.subscribe(_on_sync_settings_changed) + ) + app.state.mqtt = MqttService( db=app.state.db, provider=provider, @@ -215,21 +277,25 @@ async def _background_retention() -> None: if s.mqtt_enabled and s.mqtt_host: app.state.mqtt.start() - # Track current discovery/node so on_settings_changed can publish - # cleanup deletes against the *old* topology when those change. - app.state.mqtt._last_node_id = s.mqtt_node_id - app.state.mqtt._last_discovery_prefix = s.mqtt_discovery_prefix - def _on_mqtt_settings_change(keys, snap): # Scheduled on the running loop so async work executes safely. - asyncio.create_task(app.state.mqtt.on_settings_changed(keys, snap)) + _tasks.spawn( + app.state.mqtt.on_settings_changed(keys, snap), + name="mqtt-settings-changed", + ) - provider.subscribe(_on_mqtt_settings_change) + app.state.settings_unsubscribes.append( + provider.subscribe(_on_mqtt_settings_change) + ) try: yield finally: log.info("viofosync web UI shutting down") + # Detach settings subscribers first so the singleton provider + # doesn't keep firing into this (now tearing-down) app. + for _unsub in getattr(app.state, "settings_unsubscribes", []): + _unsub() mqtt_svc = getattr(app.state, "mqtt", None) if mqtt_svc is not None: await mqtt_svc.stop() @@ -265,7 +331,7 @@ def create_app() -> FastAPI: app = FastAPI( title="Viofosync", - version="2.2", + version="2.3", lifespan=lifespan, docs_url=None, # no swagger in prod build redoc_url=None, diff --git a/web/auth.py b/web/auth.py index 536013f..a60305f 100644 --- a/web/auth.py +++ b/web/auth.py @@ -81,8 +81,22 @@ def check_password(self, candidate: str) -> bool: def record_login_attempt(self, ip: str) -> None: """Prune old attempts and append ``now``. Raises - HTTPException 429 if the window is full.""" + HTTPException 429 if the window is full. + + NB: ``ip`` is ``request.client.host`` — behind a reverse + proxy that is the proxy's address, so the window is shared + across clients (a deliberate LAN-deployment trade-off; we + don't trust X-Forwarded-For, which is spoofable).""" now = time.monotonic() + # Sweep buckets with nothing left inside the window so the + # map can't grow unbounded from one-shot attempts by IPs that + # never return. + stale = [ + k for k, b in self._login_attempts.items() + if not b or now - b[-1] > LOGIN_WINDOW_SECONDS + ] + for k in stale: + del self._login_attempts[k] bucket = self._login_attempts.setdefault(ip, deque()) while bucket and now - bucket[0] > LOGIN_WINDOW_SECONDS: bucket.popleft() diff --git a/web/db.py b/web/db.py index 0845b76..a8cd91b 100644 --- a/web/db.py +++ b/web/db.py @@ -136,7 +136,7 @@ def migrate_legacy_db_path(new_path: str) -> None: CREATE TABLE IF NOT EXISTS export_jobs ( id INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, -- join_front|join_rear|pip + type TEXT NOT NULL, -- join_front|join_rear|pip|pip_rear|timeline clip_ids TEXT NOT NULL, -- JSON array state TEXT NOT NULL, -- queued|running|done|failed|cancelled progress REAL NOT NULL DEFAULT 0.0, @@ -234,6 +234,12 @@ def _add_column(table: str, col: str, ddl: str) -> None: _add_column("export_jobs", "clip_start", "INTEGER") _add_column("export_jobs", "clip_end", "INTEGER") + # Finished-output stats, snapshotted at finish so the export list can + # show length + size without re-probing the file on every poll (and + # even after the output is later removed). + _add_column("export_jobs", "output_size", "INTEGER") + _add_column("export_jobs", "output_duration_s", "REAL") + @contextmanager def conn(self) -> Iterator[sqlite3.Connection]: """Yield a connection with row-factory set. diff --git a/web/routers/archive.py b/web/routers/archive.py index a569f73..af76c10 100644 --- a/web/routers/archive.py +++ b/web/routers/archive.py @@ -16,17 +16,27 @@ import os from collections import defaultdict from dataclasses import dataclass -from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import FileResponse, JSONResponse, Response from ..auth import require_csrf, require_session +from ..services import durations, filmstrip, route_cache, scanner, thumbs +from ..services import tasks as _tasks from ..services import gps as gps_service -from ..services import scanner, thumbs +from ..services.naming import CHANNEL_LABELS, CHANNEL_ORDER, channel_of log = logging.getLogger("viofosync.archive") +# Until ffprobe has populated ``clip_index.duration_s`` (a background sweep +# that can take a while on a large archive), the timeline editor would see +# zero-length clips and render nothing. Fall back to the gap until the next +# clip on the same channel — dashcam clips are contiguous — capped so a +# parking gap can't produce an absurdly long block. The last clip on a +# channel has no successor to measure, so it gets a typical-clip default. +FALLBACK_MAX_S = 300.0 +FALLBACK_DEFAULT_S = 60.0 + router = APIRouter( prefix="/api/archive", tags=["archive"], @@ -52,7 +62,7 @@ def _settings(request: Request): } -def _kind_filter_clause(driving: bool, parking: bool, ro: bool) -> Optional[str]: +def _kind_filter_clause(driving: bool, parking: bool, ro: bool) -> str | None: """Build a WHERE fragment for the three event-type filters. All on → no filter. All off → ``1 = 0`` (no rows). Otherwise @@ -71,8 +81,8 @@ def _kind_filter_clause(driving: bool, parking: bool, ro: bool) -> Optional[str] @router.get("/days") def list_days( request: Request, - date_from: Optional[str] = Query(None, alias="from"), - date_to: Optional[str] = Query(None, alias="to"), + date_from: str | None = Query(None, alias="from"), + date_to: str | None = Query(None, alias="to"), driving: bool = Query(True), parking: bool = Query(True), ro: bool = Query(True), @@ -137,8 +147,8 @@ def list_days( def get_day( request: Request, date: str, - time_from: Optional[str] = Query(None), - time_to: Optional[str] = Query(None), + time_from: str | None = Query(None), + time_to: str | None = Query(None), driving: bool = Query(True), parking: bool = Query(True), ro: bool = Query(True), @@ -223,15 +233,20 @@ def _in_range(ts: int) -> bool: return {"date": date, "clips": clips} -@router.get("/day/{date}/route") -def get_route(request: Request, date: str) -> dict: - """Merged GPS track for the day plus detected journeys.""" - try: - _dt.date.fromisoformat(date) - except ValueError: - raise HTTPException(400, "bad date format") +def build_route_payload(db, recordings, date: str, geocoder) -> dict: + """Merged GPS track for a day plus detected journeys/stops, as a + JSON-able dict. Shared by GET /day/{date}/route and GET /timeline. - with _db(request).conn() as c: + The GPX re-parse is the slow part (tens of seconds on a busy day) and + only changes when the day's GPX files change, so cache it keyed by a + signature of those files. Labels are applied after, on every request, + so they stay current as the geocode cache fills. + + ``geocoder`` is the app's geocoder (or None); only its synchronous + ``cache_lookup`` is used here — uncached labels are fetched lazily + by the UI via /geocode after first paint. + """ + with db.conn() as c: rows = c.execute( """ SELECT path FROM clip_index @@ -242,14 +257,28 @@ def get_route(request: Request, date: str) -> dict: ).fetchall() gpx_paths = [r["path"] + ".gpx" for r in rows] - points, stops, journeys = gps_service.aggregate_day(gpx_paths) + sig = route_cache.signature(gpx_paths) + payload = route_cache.load(recordings, date, sig) + if payload is None: + log.info( + "route: aggregating %d GPX file(s) for %s", len(gpx_paths), date + ) + points, stops, journeys = gps_service.aggregate_day(gpx_paths) + log.info( + "route: aggregated %s -> %d point(s), %d journey(s), %d stop(s)", + date, len(points), len(journeys), len(stops), + ) + payload = _assemble_route(date, points, stops, journeys) + route_cache.store(recordings, date, sig, payload) + + _apply_labels(payload, geocoder) + return payload - # Synchronous cache lookup — no network. The UI fetches any - # uncached labels lazily via /geocode after first paint. - geocoder = getattr(request.app.state, "geocode", None) - def _lbl(lat, lon): - return geocoder.cache_lookup(lat, lon) if geocoder else None +def _assemble_route(date: str, points, stops, journeys) -> dict: + """Build the route payload (no labels — those are applied on read so + they stay current as the geocode cache fills). This is the expensive- + to-produce part that gets cached.""" return { "date": date, "point_count": len(points), @@ -263,8 +292,8 @@ def _lbl(lat, lon): "start_lon": j.start_lon, "end_lat": j.end_lat, "end_lon": j.end_lon, - "start_label": _lbl(j.start_lat, j.start_lon), - "end_label": _lbl(j.end_lat, j.end_lon), + "start_label": None, + "end_label": None, "distance_m": round(j.distance_m, 1), "duration_s": int( (j.end_time - j.start_time).total_seconds() @@ -291,13 +320,166 @@ def _lbl(lat, lon): "duration_s": int(s.duration_s), "lat": s.center_lat, "lon": s.center_lon, - "label": _lbl(s.center_lat, s.center_lon), + "label": None, } for s in stops ], } +def _apply_labels(payload: dict, geocoder) -> None: + """Fill journey/stop labels from the geocode cache (synchronous, no + network). Mutates ``payload`` in place. Uncached labels stay None and + are fetched lazily by the UI via /geocode after first paint.""" + def _lbl(lat, lon): + return geocoder.cache_lookup(lat, lon) if geocoder else None + for j in payload.get("journeys", []): + j["start_label"] = _lbl(j["start_lat"], j["start_lon"]) + j["end_label"] = _lbl(j["end_lat"], j["end_lon"]) + for s in payload.get("stops", []): + s["label"] = _lbl(s["lat"], s["lon"]) + + +@router.get("/day/{date}/route") +def get_route(request: Request, date: str) -> dict: + """Merged GPS track for the day plus detected journeys.""" + try: + _dt.date.fromisoformat(date) + except ValueError: + raise HTTPException(400, "bad date format") + geocoder = getattr(request.app.state, "geocode", None) + return build_route_payload( + _db(request), _settings(request).recordings, date, geocoder + ) + + +def _effective_durations(rows) -> dict[int, float]: + """Map clip id -> a usable duration. Uses the real probed ``duration_s`` + when present; otherwise estimates from the gap to the next clip on the + same channel (capped), so the editor renders before ffprobe catches up. + ``rows`` must be ordered by timestamp ascending.""" + by_channel: dict[str, list] = {} + for r in rows: + by_channel.setdefault(channel_of(r["camera"]), []).append(r) + + eff: dict[int, float] = {} + for chrows in by_channel.values(): + for i, r in enumerate(chrows): + real = r["duration_s"] or 0.0 + if real > 0: + eff[r["id"]] = float(real) + continue + if i + 1 < len(chrows): + gap = chrows[i + 1]["timestamp"] - r["timestamp"] + eff[r["id"]] = ( + float(min(gap, FALLBACK_MAX_S)) + if gap > 0 else FALLBACK_DEFAULT_S + ) + else: + eff[r["id"]] = FALLBACK_DEFAULT_S + return eff + + +@router.get("/timeline") +def get_timeline( + request: Request, + date: str, + journey: int | None = Query(None, ge=0), + driving: bool = Query(True), + parking: bool = Query(True), + ro: bool = Query(True), +) -> dict: + """Everything the timeline editor needs for one journey (or a whole + day when ``journey`` is omitted): channels present, clips with + channel + start_ts + duration, time bounds, and the GPS route.""" + try: + _dt.date.fromisoformat(date) + except ValueError: + raise HTTPException(400, "bad date format, use YYYY-MM-DD") from None + + log.info("timeline: open date=%s journey=%s — building route", date, journey) + geocoder = getattr(request.app.state, "geocode", None) + db = _db(request) + route = build_route_payload( + db, _settings(request).recordings, date, geocoder + ) + log.info( + "timeline: route built (%d GPS point(s)) — querying clips", + route["point_count"], + ) + + start_ts: float | None = None + end_ts: float | None = None + if journey is not None: + journeys = route["journeys"] + if journey >= len(journeys): + raise HTTPException(404, "journey index out of range") + j = journeys[journey] + start_ts, end_ts = j["start_ts"], j["end_ts"] + + where = ["group_name = ?"] + params: list = [date] + kind_clause = _kind_filter_clause(driving, parking, ro) + if kind_clause is not None: + where.append(kind_clause) + + with db.conn() as c: + rows = c.execute( + f""" + SELECT id, camera, timestamp, duration_s + FROM clip_index + WHERE {' AND '.join(where)} + ORDER BY timestamp ASC + """, + params, + ).fetchall() + + eff_dur = _effective_durations(rows) + + clips = [] + present: set[str] = set() + for r in rows: + ts = r["timestamp"] + dur = eff_dur[r["id"]] + if start_ts is not None and (ts > end_ts or (ts + dur) < start_ts): + continue + ch = channel_of(r["camera"]) + present.add(ch) + clips.append({ + "id": r["id"], + "channel": ch, + "start_ts": ts, + "duration_s": dur, + }) + + channels = [ + {"key": k, "label": CHANNEL_LABELS[k]} + for k in CHANNEL_ORDER + if k in present + ] + + if start_ts is None and clips: + start_ts = min(c["start_ts"] for c in clips) + end_ts = max(c["start_ts"] + c["duration_s"] for c in clips) + + # Each clip block lazy-loads a filmstrip sprite, so this count is how + # many ffmpeg jobs the editor may kick off — the usual cause of a NAS + # CPU spike on open. + log.info( + "timeline: date=%s journey=%s -> %d clip(s) across %d channel(s)", + date, journey, len(clips), len(channels), + ) + + return { + "date": date, + "journey": journey, + "bounds": {"start_ts": start_ts, "end_ts": end_ts}, + "channels": channels, + "clips": clips, + "gps": route if route["point_count"] > 0 else None, + } + + @router.get("/geocode") async def geocode( request: Request, @@ -317,7 +499,7 @@ async def geocode( def _fetch_clip(request: Request, clip_id: int) -> dict: with _db(request).conn() as c: row = c.execute( - "SELECT id, path, basename, size_bytes " + "SELECT id, path, basename, size_bytes, duration_s " "FROM clip_index WHERE id = ?", (clip_id,), ).fetchone() @@ -347,6 +529,41 @@ async def clip_thumb(request: Request, clip_id: int): return FileResponse(path, media_type="image/jpeg") +@router.get("/clip/{clip_id}/filmstrip") +async def clip_filmstrip(request: Request, clip_id: int): + """Slicing metadata for the clip's filmstrip sprite (generates it + on demand). 204 when ffmpeg is unavailable so the UI shows + placeholder tiles.""" + clip = _fetch_clip(request, clip_id) + s = _settings(request) + meta = await filmstrip.ensure_filmstrip( + s.recordings, clip_id, clip["path"], clip.get("duration_s") + ) + if meta is None: + return Response(status_code=204) + return { + "sprite_url": f"/api/archive/clip/{clip_id}/filmstrip.jpg", + "frames": meta.frames, + "interval_s": meta.interval_s, + "tile_w": meta.tile_w, + "tile_h": meta.tile_h, + "duration_s": meta.duration_s, + } + + +@router.get("/clip/{clip_id}/filmstrip.jpg") +async def clip_filmstrip_jpg(request: Request, clip_id: int): + clip = _fetch_clip(request, clip_id) + s = _settings(request) + meta = await filmstrip.ensure_filmstrip( + s.recordings, clip_id, clip["path"], clip.get("duration_s") + ) + sp = filmstrip.sprite_path(s.recordings, clip_id) + if meta is None or not os.path.exists(sp): + raise HTTPException(404, "no filmstrip") + return FileResponse(sp, media_type="image/jpeg") + + @router.get("/clip/{clip_id}/video") def clip_video(request: Request, clip_id: int): """Stream the MP4. ``FileResponse`` handles HTTP Range @@ -376,10 +593,13 @@ async def rescan(request: Request) -> JSONResponse: request.app.state.db, s.recordings, s.grouping, request.app.state.hub, asyncio.get_running_loop(), ) - asyncio.create_task( - scanner.sweep_missing_thumbs( - request.app.state.db, s.recordings, - ) + _tasks.spawn( + scanner.sweep_missing_thumbs(request.app.state.db, s.recordings), + name="rescan-thumb-sweep", + ) + _tasks.spawn( + durations.sweep_missing_durations(request.app.state.db), + name="rescan-duration-sweep", ) return JSONResponse({"ok": True, "indexed": n}) diff --git a/web/routers/exports.py b/web/routers/exports.py index 323b5fe..1d53bbc 100644 --- a/web/routers/exports.py +++ b/web/routers/exports.py @@ -2,16 +2,27 @@ from __future__ import annotations +import contextlib import os from typing import List from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import FileResponse, JSONResponse +from fastapi.responses import FileResponse, JSONResponse, Response from pydantic import BaseModel, Field from ..auth import require_csrf, require_session +from ..services import export_preview from ..services.naming import export_download_name, parse_clip_ids +# 1x1 transparent PNG — served when a preview can't be produced (job not done, +# unknown, or generation failed), so the degrades cleanly. +_PLACEHOLDER_PNG = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" + b"\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89" + b"\x00\x00\x00\rIDATx\x9cc\x00\x01\x00\x00\x05\x00\x01" + b"\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82" +) + router = APIRouter( prefix="/api/exports", tags=["exports"], @@ -43,11 +54,18 @@ def _resolve_default_encoder(app_state) -> str: return pref +class Segment(BaseModel): + channel: str = Field(pattern="^(front|rear|interior|other)$") + start_ts: float + end_ts: float + + class CreateExport(BaseModel): type: str = Field( - pattern="^(join_front|join_rear|pip|pip_rear)$" + pattern="^(join_front|join_rear|pip|pip_rear|timeline)$" ) - clip_ids: List[int] + clip_ids: List[int] = [] + segments: list[Segment] | None = None encoder: str | None = Field( default=None, pattern="^(software|videotoolbox|nvenc|qsv|vaapi)$", @@ -82,9 +100,13 @@ def create(body: CreateExport, request: Request) -> dict: f"encoder '{encoder}' not available on this server", ) try: - job_id = worker.enqueue( - body.type, body.clip_ids, encoder=encoder, - ) + if body.type == "timeline": + segs = [s.model_dump() for s in (body.segments or [])] + job_id = worker.enqueue_timeline(segs, encoder=encoder) + else: + job_id = worker.enqueue( + body.type, body.clip_ids, encoder=encoder, + ) except RuntimeError as e: raise HTTPException(503, str(e)) except ValueError as e: @@ -98,15 +120,26 @@ def list_jobs(request: Request) -> JSONResponse: rows = c.execute( "SELECT id, type, state, progress, error, " "created_at, started_at, finished_at, " - "clip_start, clip_end, clip_ids " + "clip_start, clip_end, clip_ids, " + "output_size, output_duration_s " "FROM export_jobs ORDER BY created_at DESC LIMIT 100" ).fetchall() + recordings = request.app.state.settings_provider.get().recordings jobs = [] for r in rows: job = dict(r) # clip_count is derived from the always-present clip_ids; the # raw id list isn't useful to the UI, so swap it out. job["clip_count"] = len(parse_clip_ids(job.pop("clip_ids"))) + # Whether the filmstrip sprite has been generated yet. The worker + # builds it after the job finishes, so a freshly-done job has none — + # the UI shows a "generating" placeholder until this flips true. + sp = export_preview.preview_path(recordings, job["id"]) + job["has_preview"] = ( + job["state"] == "done" + and os.path.exists(sp) + and os.path.getsize(sp) > 0 + ) jobs.append(job) return JSONResponse({"jobs": jobs}) @@ -151,8 +184,51 @@ def download(job_id: int, request: Request): ) +@router.get("/{job_id}/video") +def video(job_id: int, request: Request): + """Stream the export output for in-page playback. Unlike ``download`` + this sets no ``filename=`` (no attachment disposition), so a