From 08212550fc4f132028b59be9304daa19b42dacad Mon Sep 17 00:00:00 2001 From: whoisaldo Date: Wed, 26 Aug 2026 12:55:29 -0400 Subject: [PATCH 1/3] Docs and release automation for v0.2.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README rewritten for what v0.2.0 actually is: input relay, protocol v2, reliability, HEVC opt-in, honest testing story, mac dev loop, env-var reference. Badges point at CI and the latest release instead of a dead v0.1.1 asset; the hardcoded personal path is gone. - ARCHITECTURE.md rewritten around the shipped system: threading model, wire format tables, session rules, reliability mechanisms, input relay, codec negotiation, portability/testing map. - DECISIONS.md updated with the v2 rationale (clean break, FlatBuffers removal, no app-layer CRC, FEC deferred, report-as-liveness), the FFmpeg 7.1 pin, encoder-reopen model, VDD license verification (MIT — its text now ships beside the driver), input-relay design, and release policy. - RELEASE_NOTES.md gains the full v0.2.0 section. - FRIENDS_TESTING.md and QUICKSTART.txt cover the new features (input gestures, HEVC toggle, recovery behavior) and the v2 parity story. - HARDWARE_VERIFICATION.md: the ~45-minute pre-release runbook for everything CI can't prove (8 sections, per-GPU repeats, failure playbook). - release.yml: tagging v* builds the installer on CI — pinned FFmpeg 7.1, pinned Virtual-Display-Driver 25.5.2 with HARD-FAIL Authenticode verification, Inno Setup via choco, SHA-256 published in the release body (which the website already scrapes). workflow_dispatch runs the same build without publishing = the dry-run mode. - package.ps1/build-installer.ps1 honor FFMPEG_DIR (env) first and derive the version from host/Cargo.toml instead of hardcoding it; the installer script gains -StrictSignature for CI and stages the driver's license. - Website: honest claims (the "<20ms" stat and "no drivers" line are gone), v0.2.0 banner and roadmap (Extend + Control done; Audio + USB next), FFMPEG_DIR in the build-from-source snippet. - iOS MARKETING_VERSION 0.2.0, build 5. Host Cargo.toml stays 0.1.2 in this commit — the bump lands after the M8 branch merges (both touch Cargo.lock). Claude-Session: https://claude.ai/code/session_013ezpmTwAW6yAcRdy2DEaex --- .github/workflows/release.yml | 116 ++++++++++++++++ ARCHITECTURE.md | 251 ++++++++++++++++------------------ DECISIONS.md | 229 ++++++++++++++++--------------- FRIENDS_TESTING.md | 36 ++++- HARDWARE_VERIFICATION.md | 146 ++++++++++++++++++++ README.md | 227 ++++++++++++++++++------------ RELEASE_NOTES.md | 55 ++++++++ docs/download.html | 7 +- docs/index.html | 26 ++-- ios/project.yml | 4 +- scripts/QUICKSTART.txt | 15 +- scripts/build-installer.ps1 | 36 +++-- scripts/package.ps1 | 38 +++-- 13 files changed, 810 insertions(+), 376 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 HARDWARE_VERIFICATION.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..3625734 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,116 @@ +name: Release + +# Tagging vX.Y.Z builds the Windows installer and publishes a GitHub release +# with its SHA-256 in the body (docs/download.html reads both from the +# releases API). workflow_dispatch runs the same build WITHOUT publishing — +# the dry-run mode for validating this pipeline. +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: write + +env: + CARGO_TERM_COLOR: always + # The bundled Virtual Display Driver (VirtualDrivers/Virtual-Display-Driver, + # MIT). Pinned: the installer's scheduled tasks and lifecycle handling are + # verified against this exact version. + VDD_RELEASE_TAG: "25.5.2" + VDD_ASSET: "Virtual.Display.Driver-v25.05.03-setup-x64.exe" + +jobs: + windows-installer: + name: Windows installer + runs-on: windows-2022 + timeout-minutes: 60 + env: + LIBCLANG_PATH: C:\Program Files\LLVM\bin + steps: + - uses: actions/checkout@v4 + + - name: Cache FFmpeg 7.1.1 shared SDK + id: ffmpeg-cache + uses: actions/cache@v4 + with: + path: C:\ffmpeg + key: ffmpeg-7.1.1-full_build-shared + + - name: Download FFmpeg 7.1.1 shared SDK + if: steps.ffmpeg-cache.outputs.cache-hit != 'true' + shell: pwsh + run: | + $url = "https://github.com/GyanD/codexffmpeg/releases/download/7.1.1/ffmpeg-7.1.1-full_build-shared.7z" + Invoke-WebRequest -Uri $url -OutFile ffmpeg.7z + 7z x ffmpeg.7z -oC:\ffmpeg-extract | Out-Null + Move-Item C:\ffmpeg-extract\ffmpeg-7.1.1-full_build-shared C:\ffmpeg + + - name: Configure FFmpeg environment + shell: pwsh + run: | + "FFMPEG_DIR=C:\ffmpeg" | Out-File -FilePath $env:GITHUB_ENV -Append + "C:\ffmpeg\bin" | Out-File -FilePath $env:GITHUB_PATH -Append + + - uses: Swatinem/rust-cache@v2 + + - name: Download the bundled virtual display driver + shell: pwsh + run: | + $dest = "installer\vendor\vdd" + $url = "https://github.com/VirtualDrivers/Virtual-Display-Driver/releases/download/$env:VDD_RELEASE_TAG/$env:VDD_ASSET" + Invoke-WebRequest -Uri $url -OutFile (Join-Path $dest $env:VDD_ASSET) + # Its MIT license text, redistributed alongside the binary. + Invoke-WebRequest -Uri "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` + -OutFile (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") + + - name: Verify the driver's Authenticode signature + shell: pwsh + run: | + $sig = Get-AuthenticodeSignature "installer\vendor\vdd\$env:VDD_ASSET" + Write-Host "Signature status: $($sig.Status) Signer: $($sig.SignerCertificate.Subject)" + if ($sig.Status -ne "Valid") { + throw "Driver signature is '$($sig.Status)' — refusing to release an unverified driver." + } + + - name: Install Inno Setup + shell: pwsh + run: choco install innosetup -y --no-progress + + - name: Build the installer (strict driver signature) + shell: pwsh + run: .\scripts\build-installer.ps1 -StrictSignature + + - name: Compute SHA-256 + id: hash + shell: pwsh + run: | + $hash = (Get-FileHash "build\out\EternalMonitor-Setup.exe" -Algorithm SHA256).Hash + "sha256=$hash" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + Write-Host "SHA256: $hash" + + - name: Upload installer artifact (dry runs and debugging) + uses: actions/upload-artifact@v4 + with: + name: EternalMonitor-Setup + path: build\out\EternalMonitor-Setup.exe + + - name: Publish GitHub release + if: startsWith(github.ref, 'refs/tags/v') + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + $tag = $env:GITHUB_REF_NAME + $sha = "${{ steps.hash.outputs.sha256 }}" + $lines = @( + "EternalMonitor $tag — see [RELEASE_NOTES.md](https://github.com/whoisaldo/EternalMonitor/blob/main/RELEASE_NOTES.md) for what's new.", + "", + "**EternalMonitor-Setup.exe SHA-256:**", + '```', + $sha, + '```' + ) + gh release create $tag "build\out\EternalMonitor-Setup.exe" ` + --title "EternalMonitor $tag" ` + --notes ($lines -join "`n") diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 489c09f..cca7e9b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,138 +1,125 @@ # EternalMonitor — Architecture -## Current pipeline +Accurate as of v0.2.0 (protocol v2). + +## The pipeline ```text -[Windows desktop] - | - v -+---------------------------------------------+ -| host/ (Rust Windows app) | -| | -| DXGI Desktop Duplication | -| -> primary display capture | -| -> CPU-readable BGRA frame | -| | -| ffmpeg-next + NVENC | -| -> BGRA to YUV420P | -| -> H.264 encode | -| | -| UDP transport | -| -> FlatBuffer FramePacket | -| -> custom fragmentation header | -| -> mDNS advertisement | -+---------------------------------------------+ - | - v -+---------------------------------------------+ -| ios/ (Swift iPad app) | -| | -| Connect UI | -| -> manual IP entry | -| -> Bonjour scan attempt | -| | -| UDP receiver | -| -> fragment reassembly | -| -> FramePacket parse | -| | -| VideoToolbox | -| -> H.264 decode | -| | -| Metal MTKView | -| -> render latest decoded frame | -+---------------------------------------------+ +[Windows desktop] [iPad] + | ^ + v | Metal (NV12 + BT.601/709 shader, ++-------------------------- host/ ---------------|-- aspect-fit, draw-on-demand) +| capture thread encode thread | VideoToolbox (H.264/HEVC, hw on +| DXGI duplication --> BGRA->YUV420P --> | device, sw in the simulator) +| (or synthetic) slot swscale, then | FrameAssembler (per-frame +| cursor composite NVENC/AMF/QSV/x264/ | fragment reassembly, caps) +| x265/VideoToolbox | UDPReceiver (ephemeral port, +| | | media/control demux) +| v channel | +| transport task (tokio) -----+--> media datagrams (UDP) +| v2 fragmentation, pacing, <--- control datagrams (same socket) +| session, heartbeats, ABR | ++-------------------------------------------------+ + supervised by supervisor.rs (health, watchdogs, backoff restarts) ``` -## What is implemented - -### Capture - -- API: DXGI Desktop Duplication -- Current behavior: enumerates all adapters/outputs and duplicates a **selectable** output - (primary by default), copying it into a CPU-readable staging texture -- The Settings tab exposes a capture-display picker; choosing a virtual output created by - an Indirect Display Driver turns the iPad into an extended desktop instead of a mirror. - The capture adapter follows the chosen output; encoder selection stays vendor-based. -- The managed virtual display is brought up **on demand, only once an iPad has connected**, and - is disabled on exit / target change / startup (and via a panic hook), so it never lingers as a - phantom monitor. If the captured display is idle, the loop resends the last frame so the iPad - still receives a startup keyframe. -- Output format passed downstream: BGRA frame buffer plus frame metadata - -This is functional but not yet the final zero-copy path described in earlier docs. - -### Encode - -- Crate: `ffmpeg-next` -- Codec path: `h264_nvenc` -- Current codec settings: - - H.264 - - `baseline` profile - - `gop=30` - - `max_b_frames=0` - - `zerolatency=1` - - `rc=cbr` - -The encoder emits Annex B H.264 byte streams that are wrapped in `FramePacket`. - -### Transport - -- Current transport: WiFi/local-network UDP only -- Registration handshake: iPad sends `ETERNALHELLO` plus its listen port -- Packetization: - - each encoded frame becomes one FlatBuffer `FramePacket` - - payload is fragmented into MTU-sized UDP datagrams - - fragment header is currently `16` bytes - - fragment index and fragment count are `u16` - - the header's final 4 bytes carry a per-pipeline-run `stream_epoch`, so the receiver drops - stale reassembly state immediately on a stream restart (seq reset) instead of inferring it - from a sequence gap. These bytes were previously reserved/zero, so older receivers that - ignore them stay wire-compatible. - -The `u16` fragment-count change is important. Older host and iPad builds are not wire-compatible with the current transport fix. - -### Discovery - -- Host side: advertises `_eternaldisplay._udp.local.` over mDNS/DNS-SD -- iPad side: scans via `NetServiceBrowser` - -This exists in code, but it is not reliable enough to treat as finished. Direct IP connect is the known-good path. - -### iPad receive/decode/render - -- UDP datagrams are received with `NWConnection` -- Fragments are reassembled by sequence number -- `FramePacket` is parsed manually from FlatBuffers -- H.264 is decoded with `VTDecompressionSession` -- Frames are rendered with `MTKView` - -The renderer currently keeps the latest available decoded texture and draws that. - -## Protocol - -Implemented message shape in active use: - -- `FramePacket` - - `seq` - - `timestamp_us` - - `data` - - `width` - - `height` - - `is_keyframe` - -Not all planned protocol families are implemented yet. `InputEvent`, richer control messages, and display configuration exchange are still roadmap items. - -## Known-good state - -- Working end-to-end UDP stream: commit `bc44770` -- Manual IP connect works -- Network scan/discovery may still fail even when streaming works - -## Not implemented yet - -- First-party signed display driver (the host currently drives a third-party signed - virtual display driver; a first-party in-tree IDD is a v0.2.0 goal) -- USB transport -- Reverse input channel -- Reliability controls such as selective NACK -- Dynamic transport switching +Host stages are dedicated OS threads connected by a **latest-wins frame slot** +(capture → encode: an unconsumed frame is displaced, so the encoder always +works on the freshest picture) and a **lossless channel** (encode → transport: +a dropped encoded P-frame would corrupt the GOP). Frame pixels travel in +`Arc>` buffers that are recycled — steady state does one full-frame +copy (the DXGI staging readback). + +The **supervisor** owns the pipeline: stage threads report their exit, wedge +watchdogs fire on silent stalls (loop heartbeat stale 3 s, no frame 5 s with a +client, encoder flat 3 s), and restarts get exponential backoff with a +restart-storm brake. The client **session lives outside the pipeline**, so a +crash-restart resumes streaming to the same session — the client sees a new +stream epoch and resets reassembly, with no re-handshake. + +## Wire protocol v2 + +One UDP socket carries both media and control. Every datagram starts with an +8-byte prefix: magic `"EM"`, version `2`, packet type, flags (media bit 0 = +keyframe), reserved, and a strict payload length. Legacy v1 datagrams began +with `"ET"`, so the two are unambiguous; v2 is otherwise a **clean break** +(each side tells the user to update the other on contact with v1). + +**Media** (type 0x01): a 32-byte header — session id, stream epoch, frame +sequence, fragment index/count (≤3066 ≈ 4 MiB per frame), capture timestamp +(µs on the host process clock) — followed by a raw Annex B chunk. No +serialization framework; width/height/codec travel in the control plane. + +**Control** (16-byte header: session id, message sequence, type): + +| Message | Direction | Purpose | +| --- | --- | --- | +| HELLO2 / HELLO_ACK | C→H / H→C | Session establishment: capability bits (H.264/HEVC decode, wants-input), screen size/refresh, nonce-idempotent ACK carrying session id, host-dictated timing, and the stream config | +| HEARTBEAT | H→C | 1 Hz liveness + embedded stream config (self-heals lost config changes) | +| RECEIVER_REPORT | C→H | 500 ms cadence: loss, completion, jitter, depths — feeds ABR and doubles as client liveness | +| KEYFRAME_REQUEST | C→H | Loss/decode-error recovery; host rate-limits to 1 per 500 ms | +| PING / PONG | C→H→C | NTP-style clock sync (min-RTT offset) for the honest end-to-end latency readout | +| STREAM_CONFIG | H→C | Immediate notify on bitrate/codec/resolution change | +| INPUT_EVENT | C→H | Input relay (below) | +| BYE | both | Clean teardown with a reason (user, backgrounded, shutdown) | + +**Session rules** (host, pure state machine): one client at a time — a second +device gets `busy`; the same device reconnecting supersedes in place with a +fresh session id; duplicate HELLO2 nonces get an identical ACK (retransmit +tolerance); liveness expires 3 s after the last report/input, which also +tears down the virtual display. + +## Reliability + +- **ABR**: a bitrate ladder (4–20 Mbps, capped by the GUI "Max bitrate" + slider) driven by receiver reports — loss or PLI pressure steps down + (cooldown 3 s), 15 s of clean reports steps back up. A bitrate change + reopens the encoder session (~50–200 ms, same epoch) and forces an IDR. +- **Pacing**: keyframe bursts are sent in batches with microsleeps under a + hard 3 ms wall-clock budget, plus a 4 MiB kernel send buffer — smoothing + WiFi loss without adding latency. +- **Recovery**: the client requests keyframes on gaps/decode errors; the host + honors at most 2/s. The app shows "SIGNAL LOST" after 3 s of silence and + auto-reconnects with backoff. + +## Input relay + +The iPad normalizes touches over the displayed video (letterbox-corrected) to +a 0–65535 grid. A pure gesture machine turns raw touches into events: tap = +click at the touch-down point, drag commits after slop, two fingers scroll +(direct-manipulation direction), a 500 ms hold right-clicks, Apple Pencil +presses immediately with pressure. Press/release edges are sent twice with one +event id; the host dedupes, maps through the captured output's desktop +rectangle onto the Windows virtual screen, and injects with `SendInput`. +Sessions that didn't set the wants-input capability are never injected for. + +## Codecs + +H.264 is the default everywhere. With the host's "Prefer HEVC" setting on and +a client that advertises HEVC decode, the encoder live-switches to the HEVC +sibling (NVENC/AMF/QSV/VideoToolbox/x265) via the same reopen mechanism. The +iPad decoder detects the codec **from the bitstream** (an HEVC VPS in a +keyframe), so the switch has no config/media ordering race. HEVC encoders are +configured to repeat VPS/SPS/PPS in-band; H.264 keyframes get cached SPS/PPS +prepended (the AMF path has extra guards developed against real hardware). + +## Discovery + +The host advertises `_eternaldisplay._udp` over mDNS with `version`, +`proto=2`, and `platform` TXT records, re-upserted every 60 s (no +unregister gap) and withdrawn with goodbye packets on exit. Manual IP entry +and the QR code remain the fallback for networks that filter multicast. + +## Portability and testing + +Everything protocol- or logic-shaped lives in the pure `eternal-wire` crate +(v2 codecs, H.264/HEVC bitstream helpers, reassembly) or in host modules with +injected clocks (session, ABR, pacer, supervisor, input mapping) — all tested +on every platform. Windows-only code (DXGI, SendInput, VDD control) is +`cfg(windows)`-gated; a synthetic capture source with a machine-readable +frame counter lets the full host run on macOS/CI, where the Rust E2E suite +drives a fake receiver through handshake, loss, crash-recovery, input, and +HEVC scenarios, and `scripts/e2e_ios.sh` streams into the real iPad app in +the simulator. Golden wire vectors are parsed byte-for-byte by both +languages. What only hardware can prove (GPU encoders, the display driver, +real WiFi) is enumerated in the release hardware runbook. diff --git a/DECISIONS.md b/DECISIONS.md index f11fcae..bced713 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -8,131 +8,146 @@ - Works well with `tokio`, `windows`, and `ffmpeg-next` - Keeps the host codebase native and low-level without switching to C++ -Rejected: - -- C++: more binding and build overhead for this repo shape -- C#: less suitable for the current native graphics and transport path +Rejected: C++ (more binding/build overhead for this repo shape), C# (wrong +fit for the native graphics + transport path). ### Swift for the iPad app - Direct access to VideoToolbox, Metal, Network, and UIKit/SwiftUI APIs -- Best fit for hardware decode and native iPad rendering - -Rejected: -- React Native / Flutter: wrong fit for this decode/render stack +Rejected: React Native / Flutter — wrong fit for this decode/render stack. ## Capture -### DXGI Desktop Duplication - -Chosen because it is the practical Windows desktop capture API for this project. - -Current reality: - -- It is working -- The current implementation copies into a CPU-readable staging texture -- Dirty rect metadata is queried, but the pipeline does not yet exploit it for partial encode +### DXGI Desktop Duplication, CPU readback -Earlier docs overstated this as a never-touch-CPU path. That is not true in the current build. +The practical Windows desktop-capture API. The pipeline does one full-frame +copy (staging-texture readback into a recycled `Arc` buffer); cursor +compositing is CPU-side. Dirty-rect metadata is queried but not yet used for +partial encode. A zero-copy GPU path (capture texture straight into the +encoder) remains future work — measure first: after the v0.2.0 hot-path work, +the readback is no longer the dominant cost at 1080p60. ## Encoder -### `ffmpeg-next` with multi-vendor hardware encode - -The host detects the GPU vendor via DXGI adapter enumeration and selects the best -available hardware encoder using a vendor-preferred fallback chain: - -1. Vendor-preferred encoder (NVENC for NVIDIA, AMF for AMD, QSV for Intel) -2. Other hardware encoders in order: NVENC → AMF → QSV -3. Software fallback: libx264 - -Each encoder has tuned low-latency options (`gpu.rs` resolves the encoder, -`encoder/mod.rs` applies per-encoder settings). The GPU with the most dedicated -VRAM is selected automatically; software adapters are excluded. - -Current reality: - -- NVIDIA (h264_nvenc), AMD (h264_amf), Intel (h264_qsv), and software (libx264) paths are implemented -- H.265 is not implemented in the active iPad path yet - -### H.264 Baseline - -Chosen for the current iPad decode path because it is the simplest compatibility target for VideoToolbox bootstrap and stream startup. - -## Transport - -### Custom UDP framing - -Chosen because the project cares more about low latency than guaranteed in-order delivery. - -Current reality: - -- The repo currently implements custom UDP fragmentation and reassembly -- There is no selective NACK layer yet -- There is no USB transport yet -- The active wire format uses a `16` byte fragment header with `u16` fragment index/count fields -- The header's final 4 bytes carry a per-pipeline-run `stream_epoch` so the receiver detects a - stream restart immediately; the bytes were previously reserved/zero, so older receivers that - ignore them remain wire-compatible - -That `u16` change fixed large-frame corruption where `fragment_count` overflowed at `255`. - -Rejected for now: - -- TCP: head-of-line blocking is the wrong tradeoff here -- WebRTC: too heavy for the current stage -- RTP: more complexity than the current implementation needs - -## Protocol - -### FlatBuffers - -Chosen because the host and iPad both need a compact binary packet format with clear field structure. - -Current reality: - -- `FramePacket` is in active use -- Swift currently uses a manual parser matched to the Rust serializer -- The broader protocol families described in older planning docs are not all implemented yet +### `ffmpeg-next` pinned to FFmpeg 7.1, multi-vendor hardware encode + +The host detects the GPU vendor (DXGI adapter enumeration) and walks a +vendor-preferred chain — NVENC / AMF / QSV, then libx264 — with per-encoder +low-latency options. FFmpeg is pinned to **7.1** everywhere (Windows DLLs, +CI's downloaded SDK, Homebrew `ffmpeg@7`) so the bytes tested are the bytes +shipped; upgrading to 8.x is deliberately a separate change now that the E2E +harness exists to validate it. + +AMF is the fragile path (startup keyframes without parameter sets, strict +VideoToolbox level requirements) and carries bespoke guards developed against +real hardware; those guards are preserved verbatim through refactors. + +### H.264 baseline default, HEVC opt-in + +Baseline H.264 is the simplest compatibility target for VideoToolbox startup. +HEVC ships behind a host setting ("Prefer HEVC") until it has been proven on +each hardware encoder — negotiation requires both the setting and the +client's advertised decode capability, and any HEVC open failure falls back +to H.264 silently. + +### Encoder reconfiguration = session reopen + +Hardware encoders ignore bitrate pokes on an open context, so every real +change (ABR rung, slider, codec switch) reopens the encoder session (~50–200 +ms, same stream epoch) and forces an IDR. The old per-frame `apply_bitrate` +call — which silently did nothing on NVENC/AMF — is gone; the GUI value is +now the ABR **ceiling**, labeled accordingly. + +## Transport & protocol + +### Protocol v2: raw Annex B over a custom UDP framing, one socket for media + control + +v2 replaced the v1 format (FlatBuffers `FramePacket` + 16-byte fragment +header + fire-and-forget `ETERNALHELLO`) as a **clean break** — both sides +ship together, and each side recognizes the other's legacy traffic well +enough to say "update the other half". Rationale for the break: the v1 +format had no version field, no session identity, and no back channel, so +compatible evolution wasn't possible. + +- **FlatBuffers removed**: the only dynamic field was the frame payload + itself; width/height/codec belong to the control plane (STREAM_CONFIG), + so media is now a fixed 32-byte header + raw Annex B. One less + serialization layer on the per-frame hot path, one less parser to fuzz. +- **No app-layer checksum/CRC**: UDP's checksum plus magic/version/session + checks and strict length validation catch stray and truncated datagrams; + a corrupted-but-valid datagram costs at most an artifact until the next + keyframe, which the PLI path requests anyway. A CRC would tax every packet + to protect against the rarest failure with the mildest consequence. +- **FEC deferred**: consumer-WiFi loss is bursty, which single-XOR parity + handles poorly. Pacing + keyframe recovery + ABR carry v0.2.0; packet + types and flag bits are reserved for FEC if measurement ever justifies it. +- **Client liveness = its receiver reports** (500 ms cadence) rather than a + dedicated client heartbeat — the reports must flow anyway to drive ABR. + +Rejected: TCP (head-of-line blocking), WebRTC (too heavy), RTP/RTCP (we'd +use a fraction of it and still need custom extensions for input/config). + +### Adaptive bitrate on the host, signals from the client + +The host owns the ladder (it owns the encoder); the client just reports +honestly. Loss or keyframe-request pressure steps down, sustained clean +reports step up, and the GUI slider caps the ladder. ## Discovery -### mDNS/DNS-SD - -Chosen for zero-config host discovery on local networks. +### mDNS/DNS-SD, best effort, manual IP as the guaranteed path -Current reality: - -- The host advertises a Bonjour service -- The iPad scans for that service -- Direct IP connect is more reliable than discovery at the moment - -So discovery is still an incomplete feature, not something to depend on. +The host advertises `_eternaldisplay._udp` (TXT: `version`, `proto=2`, +`platform`), re-upserting every 60 s — the v0.1.x unregister-then-register +refresh created a periodic discovery hole — and sends goodbye packets on +exit. Multicast-filtering networks still exist, so manual IP + QR remain +first-class. ## Virtual display ### Bundled third-party Indirect Display Driver, managed on demand -The extended-display feature drives the bundled VirtualDrivers/Virtual-Display-Driver. The host -does not run elevated, so the installer registers two SYSTEM scheduled tasks (enable/disable) that -the host triggers via `schtasks /Run` — no per-toggle UAC prompt. - -Current reality: - -- The device is left disabled by default and is enabled **only once an iPad connects**, then - disabled on exit / target change / startup and via a panic hook — so it never lingers as a - phantom monitor. -- The tasks resolve the device at trigger time (name-agnostic) rather than baking a fixed - instance id, so they survive driver-version and PnP-enumeration differences. - -A first-party signed display driver (removing the third-party dependency) is still a v0.2.0 goal. - -## Deferred decisions - -These remain intentionally unresolved until the current WiFi UDP path is hardened: - -- USB transport design -- First-party signed display driver (today the host manages a bundled third-party VDD) -- Input relay protocol and host injection mechanism -- Idle-disconnect teardown of the virtual display (needs a bidirectional iPad heartbeat) +The extended display drives the bundled +[VirtualDrivers/Virtual-Display-Driver](https://github.com/VirtualDrivers/Virtual-Display-Driver) +(**MIT licensed — verified; its license text ships in the installer next to +the driver**). The host doesn't run elevated, so the installer registers two +SYSTEM scheduled tasks (enable/disable) the host triggers via `schtasks` — +no per-toggle UAC. + +- Enabled **only while an iPad is connected**; disabled on exit, target + change, startup, panic, and now on client loss (the v2 liveness signal + finally made idle teardown possible). +- Before enabling, the host writes `vdd_settings.xml` so the virtual display + offers the iPad's native landscape resolution and refresh (opt-out in + Settings). +- A first-party signed display driver remains a long-term goal. + +## Input relay + +### Wire normalized coordinates; a pure gesture machine on the client; `SendInput` on the host + +Touches are normalized over the *displayed video* (letterbox-corrected), so +the wire format is resolution-independent. Tap-vs-drag-vs-scroll +disambiguation is a pure state machine (tap = click on release; drag commits +after slop; pencil presses immediately because ink can't wait). Edges are +sent twice with one event id and deduped host-side — loss tolerance without +retransmit machinery. Injection maps through the captured output's desktop +rectangle, so multi-monitor and virtual-display layouts land clicks on the +right screen. + +## Versioning & release + +- One version, single-sourced from `host/Cargo.toml` (`env!` into the banner + and the mDNS TXT), matched by the iOS `MARKETING_VERSION`. +- Releases are built by CI from a `v*` tag: pinned FFmpeg, pinned driver + version, **hard-fail Authenticode verification** on the bundled driver, + and the installer's SHA-256 published in the release body (the website + reads it from there). + +## Deferred + +- Audio (WASAPI → Opus → AVAudioEngine sketch exists; wire types reserved) +- USB transport +- First-party signed display driver +- FEC (types/flags reserved), zero-copy GPU capture, dirty-rect encode diff --git a/FRIENDS_TESTING.md b/FRIENDS_TESTING.md index 878bca0..3a9dc0d 100644 --- a/FRIENDS_TESTING.md +++ b/FRIENDS_TESTING.md @@ -26,10 +26,29 @@ so its display task is registered. ## Build parity matters -The host and iPad app must be from the **same release**. The v0.1.1 transport changed the -fragment count to `u16` (see `ARCHITECTURE.md`), so an older iPad build talking to a newer -host (or vice-versa) shows corrupted or no video. When inviting a tester, give them the -matching iPad TestFlight build and the matching Windows zip together. +The host and iPad app must be from the **same release**. v0.2.0's protocol v2 +is a deliberate clean break: a v0.1.x app meeting a v0.2.0 host (or the +reverse) won't stream — each side shows an explicit "update the other half" +message instead of corrupted video, so at least the failure is obvious. Hand +out the matching TestFlight build and installer together. + +## New in v0.2.0 — things worth testing on purpose + +- **Input relay**: tap/drag/two-finger scroll/hold-for-right-click/Pencil + should feel like a trackpad. Check multi-monitor setups (clicks must land + on the captured screen) and the "Control PC with touch" toggle off (host + must ignore touches). +- **HEVC**: flip "Prefer HEVC" in host Settings mid-stream; the codec on the + iPad's Settings HOST module should flip to HEVC within a second, and back. + If video breaks only in HEVC on some GPU, that encoder's HEVC path is the + bug — collect logs and turn the toggle off. +- **Recovery**: kill the host mid-stream (Task Manager) and relaunch — the + iPad should show SIGNAL LOST and reconnect by itself. Walk to the edge of + WiFi range — the picture should coarsen (bitrate stepping down) rather + than freeze, and recover afterwards. +- **Extended display resolution**: with "Match extended display to the + iPad's resolution" on (default), the virtual display should come up at the + iPad's native aspect — no letterboxing on the iPad. ## Cover all three GPU vendors @@ -71,6 +90,9 @@ now shows an **amber warning banner** on the Stream tab. CPU encoding is hot and ## Known limitations (so you don't chase ghosts) -No NACK/retransmit, no congestion control, no jitter buffer (see `ARCHITECTURE.md` → -"Not implemented yet"). On a clean network the stream is smooth; on a lossy one it will drop -frames with no recovery. That's expected for this build, not a regression. +No audio, no USB transport, and no retransmit of lost packets — loss shows as +a brief artifact or frame skip, then the stream self-heals with a requested +keyframe (and steps the bitrate down if loss persists). Sustained stutter on +a clean network IS reportable now; on hotel/guest WiFi it's still the +network. HEVC is experimental and off by default — if a stream misbehaves, +confirm the codec on the Stream tab before filing it as a general bug. diff --git a/HARDWARE_VERIFICATION.md b/HARDWARE_VERIFICATION.md new file mode 100644 index 0000000..9040113 --- /dev/null +++ b/HARDWARE_VERIFICATION.md @@ -0,0 +1,146 @@ +# v0.2.0 Hardware Verification Runbook + +What CI cannot prove: real GPU encoders, the virtual display driver, real +WiFi, and touch feel. Run this on the Windows PC + a real iPad before +tagging the release. Budget ~45 minutes (plus per-GPU repeats if you can +borrow AMD/Intel machines). Check items off; anything that fails gets logs +(see the last section) and a fix round before the tag. + +Conventions: **Expect** is the pass condition. `Host log` = Copy logs button +(Stream tab) or `%APPDATA%\EternalMonitor\logs\eternal-host-session.log`. + +## A. Install & first light (5 min) + +- [ ] **A1 — Installer**: run `EternalMonitor-Setup.exe` (SmartScreen → "Run + anyway", one UAC prompt). *Expect*: install completes, host launches, no + second UAC. Upgrading over a previous version keeps settings. +- [ ] **A2 — Firewall**: on first run tick BOTH Private and Public. + *Expect*: prompt appears exactly once. +- [ ] **A3 — Banner**: host log shows `EternalMonitor v0.2.0`, the right GPU + name, and a hardware encoder (not x264) with no fallback banner in the GUI. + +## B. Basic streaming (8 min) + +- [ ] **B1 — Manual IP connect**: iPad → enter the host IP. *Expect*: + picture in under 2 s, iPad HUD ~60 fps, host Stream tab shows the client. +- [ ] **B2 — Discovery**: iPad Scan finds the host. Leave the scan list open + 4+ minutes (crosses two 60 s re-advertisements). *Expect*: the host never + blinks out of the list. Quit the host app. *Expect*: it leaves the list + within a few seconds (mDNS goodbye), not after minutes. +- [ ] **B3 — QR connect**: scan the host's QR from the iPad. *Expect*: + connects to the same address the GUI shows. +- [ ] **B4 — Truthful readouts**: host Stream tab codec matches the iPad + Settings → HOST module (name, resolution, fps, codec, bitrate), and the + HOST bitrate follows the ABR rung, not just the slider. +- [ ] **B5 — Latency sanity**: drag a window in circles; the iPad HUD's ms + readout should sit in the tens (typically 20–80 ms on good WiFi) and the + motion should feel attached. If you have a 240 fps camera, film both + screens and count frames — HUD claim within ~±20 ms of measured. +- [ ] **B6 — Decoder**: iPad Settings diagnostics say "hardware decoder" + (the simulator's software path must not appear on device). + +## C. Input relay (7 min) + +- [ ] **C1 — Click targets**: tap small targets (window close buttons) in + all four screen corners. *Expect*: exact hits — no offset (this validates + desktop-rect mapping; test at 100% AND at 150% display scaling). +- [ ] **C2 — Drag**: drag a window smoothly; text selection works; no + spurious clicks when starting a two-finger scroll. +- [ ] **C3 — Scroll**: two-finger scroll in a browser. *Expect*: content + follows the fingers (direct-manipulation direction), smooth, both axes. +- [ ] **C4 — Right-click**: hold ~½ s. *Expect*: context menu at the touch + point. Tap elsewhere dismisses it (single click, not double). +- [ ] **C5 — Pencil**: in Paint/whiteboard, ink starts immediately on + contact (no tap-vs-drag delay) and pressure varies the stroke where + supported. +- [ ] **C6 — Multi-monitor**: with a second physical monitor attached, + capture monitor 2 — touches must land on monitor 2, never the primary. +- [ ] **C7 — View-only**: turn "Control PC with touch" off, reconnect. + *Expect*: touches do nothing on the PC; single-tap toggles the HUD. + +## D. Reliability (8 min) + +- [ ] **D1 — Host death**: kill the host from Task Manager mid-stream. + *Expect*: iPad shows SIGNAL LOST within ~3 s. Relaunch the host. + *Expect*: the iPad reconnects by itself (no taps) within ~10 s. +- [ ] **D2 — ABR under real loss**: walk toward the edge of WiFi range (or + microwave the link). *Expect*: picture softens (host GUI bitrate steps + down), no multi-second freezes; walking back sharpens it within ~20 s. +- [ ] **D3 — Live bitrate change**: move the Max-bitrate slider mid-stream. + *Expect*: a sub-second hiccup at most, no disconnect, no epoch weirdness. +- [ ] **D4 — Backgrounding**: swipe the app away to the switcher. *Expect*: + host Stream tab returns to "waiting for client" within ~3 s (BYE), and if + streaming the virtual display it tears down. Reopen the app. *Expect*: + auto-resume ("Resume after switching apps" default on). +- [ ] **D5 — Second device busy**: while one iPad streams, connect from a + second (or the simulator). *Expect*: clear "host is busy" message; the + first stream is untouched. +- [ ] **D6 — Version mismatch UX** (if a v0.1.x build is still around): + old app → new host and new app → old host each show an explicit "update + the other side" message, not garbage video. + +## E. HEVC (5 min, repeat per GPU vendor available) + +- [ ] **E1 — Switch on**: mid-stream, tick "Prefer HEVC". *Expect*: iPad + HOST module codec flips to HEVC within ~1 s, picture stays clean, no + reconnect. Untick → back to H.264 the same way. +- [ ] **E2 — Quality/limits**: at the same bitrate HEVC should look no worse + than H.264. Watch 2+ minutes for artifacts, especially on **AMF** (its + `header_insertion_mode` handling is the least-proven path). Any breakage: + note GPU + driver version, collect logs, and leave the toggle off. +- [ ] **E3 — Fallback**: on a GPU without an HEVC encoder, the toggle warns + once in the log and keeps streaming H.264 (no error loop). + +## F. Extended display & resolution match (8 min) + +- [ ] **F1 — Lifecycle**: select "Extended display (iPad)" + Restart stream + with the iPad connected. *Expect*: a new display appears in Windows + Display settings, windows drag onto it, and the iPad shows it. Disconnect + the iPad. *Expect*: the virtual display disappears within ~5 s + (client-lost teardown). Quit/relaunch/crash never strands a phantom + monitor. +- [ ] **F2 — Native resolution**: with "Match extended display to the iPad's + resolution" on (default), the virtual display's mode equals the iPad's + native landscape resolution (e.g. 2420×1668) — the iPad picture is + edge-to-edge, no letterbox. Check + `C:\VirtualDisplayDriver\vdd_settings.xml` exists and lists that mode + first. Toggle the match off + restart. *Expect*: driver default mode + (letterboxed picture is fine here). +- [ ] **F3 — 120 Hz mode** (ProMotion iPad): with the match on, Windows + offers the panel refresh (or falls back to the 60 Hz variant without + erroring). + +## G. Encoder deep checks (per vendor; ~5 min each) + +- [ ] **G1 — Idle VBV (real PTS)**: leave a static desktop for 60 s. + *Expect*: bandwidth on the Performance tab collapses (keepalive only), and + the first motion afterwards is clean, not a smear. If pacing looks wrong + on NVENC/AMF, retry with `set ETERNAL_LEGACY_PTS=1` and report — that + escape hatch existing is why this item is here. +- [ ] **G2 — AMF specifics** (AMD box): startup shows a keyframe (no black + screen), recovery after loss works, and 10 minutes of streaming shows no + periodic freeze. If broken: `set ETERNAL_AMF_DIAG=1`, reproduce, send + `%APPDATA%\EternalMonitor\diagnostics\`. +- [ ] **G3 — High refresh**: `set ETERNAL_FPS=120` on a ProMotion iPad. + *Expect*: HUD ~100+ fps on a strong network, no capture-side stutter. +- [ ] **G4 — Stop/Start**: GUI Stop then Start. *Expect*: clean halt and a + fresh stream the iPad resumes automatically. + +## H. Long soak (run in the background of the above) + +- [ ] **H1**: keep one stream up 30+ minutes. *Expect*: no leak-shaped + memory growth on either end (Task Manager / Xcode gauge), no thermal + shutdown of the stream, HUD stats stay sane. + +## When something fails + +1. Host: **Copy logs** (Stream tab) or grab + `%APPDATA%\EternalMonitor\logs\eternal-host-session.log`. +2. iPad: Settings → the diagnostics list (most recent events), plus what the + screen showed. +3. Note GPU model + driver version, WiFi band, and which runbook item. +4. AMD encode issues: also `%APPDATA%\EternalMonitor\diagnostics\` with + `ETERNAL_AMF_DIAG=1` set. + +Fixes land, the failing items get re-run, and only then does `v0.2.0` get +tagged (the tag builds and publishes the installer automatically). diff --git a/README.md b/README.md index 6c152ec..93ffdf7 100644 --- a/README.md +++ b/README.md @@ -1,128 +1,177 @@ # EternalMonitor -## ⚡ Contribute -**OPEN TO CONTRIBUTIONS.** +Use your iPad as a wireless second display for Windows — and control the PC from it. -Whether you want to optimize the network transport layer, refine the Metal rendering pipeline, or improve Windows capture efficiency, your PRs are highly welcome. - -Any help would be appreciated! :blush: - -> 💬 **Help build this together. Ping me directly on Discord to collaborate:** **`aldobenches285`** - -[![Version](https://img.shields.io/badge/version-v0.1.2--mirror-e8ff47?style=flat&labelColor=111)](https://github.com/whoisaldo/EternalMonitor/releases/tag/v0.1.2-mirror) -[![Download](https://img.shields.io/badge/download-installer-blue?style=flat&labelColor=111)](https://github.com/whoisaldo/EternalMonitor/releases/download/v0.1.2-mirror/EternalMonitor-Setup.exe) +[![CI](https://github.com/whoisaldo/EternalMonitor/actions/workflows/ci.yml/badge.svg)](https://github.com/whoisaldo/EternalMonitor/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/whoisaldo/EternalMonitor?labelColor=111&color=e8ff47)](https://github.com/whoisaldo/EternalMonitor/releases/latest) [![Website](https://img.shields.io/badge/website-eternalmonitor.dev-e8ff47?style=flat&labelColor=111)](https://eternalmonitor.dev) -**Website:** [eternalmonitor.dev](https://eternalmonitor.dev) - -Use your iPad as a low-latency Windows display receiver over local-network UDP. - -This repo currently contains a working Windows host capture/encode/transport path and a working iPad receive/decode/render path. The known-good stream state is commit `bc44770` on branch `feature/rust-workspace-bootstrap`. - -## Current status - -Implemented now: - -- Windows host captures a **selectable display output** (primary by default) with DXGI Desktop Duplication -- Capture-display picker in the Settings tab — stream any output, including a **virtual extended display** created by a signed Indirect Display Driver, so the iPad can be a true second screen instead of only a mirror. The virtual display is brought up **on demand, only while an iPad is connected**, and removed on exit — no phantom monitor when idle -- Host converts BGRA to YUV420P and can select hardware H.264 encoders per GPU vendor -- Host advertises an mDNS/DNS-SD service and streams frames over UDP on port `9876` -- iPad app receives fragmented UDP datagrams, reassembles `FramePacket` payloads, decodes with VideoToolbox, and renders with Metal -- Direct connect by IP works end to end with the current transport format -- One-step Windows installer (`EternalMonitor-Setup.exe`) that bundles the host, FFmpeg runtime, and the virtual display driver for non-technical testers - -Not done yet: +A Rust host on the PC captures the desktop with DXGI, encodes with the GPU +(NVENC/AMF/QSV, H.264 or opt-in HEVC), and streams over UDP on the local +network. A native Swift app on the iPad decodes with VideoToolbox and renders +with Metal. Touch, Apple Pencil, and two-finger scroll are relayed back as +mouse input. MIT licensed. + +**Contributions welcome** — transport, encoders, rendering, docs, anything. +Ping `aldobenches285` on Discord to collaborate. + +## What v0.2.0 does + +- **Mirror or extend**: mirror the primary display, capture a specific + monitor, or stream a managed *virtual* extended display that exists only + while the iPad is connected — offered at the iPad's native resolution and + refresh rate. +- **Control the PC from the iPad**: tap to click, drag to move the mouse, two + fingers to scroll, hold for a right-click, Apple Pencil with pressure. + Negotiated per session; a view-only toggle is one switch away. +- **Protocol v2**: a real session (handshake with capability negotiation, busy + rejection, liveness), host heartbeats, client keyframe requests, receiver + reports, and NTP-style clock sync for an honest end-to-end latency readout. +- **Reliability**: adaptive bitrate (the host slider is the ceiling), packet + pacing on keyframe bursts, keyframe recovery after loss, automatic + reconnect after signal loss, and supervisor-driven crash recovery on the + host (an encoder crash restarts the pipeline in ~1 s without dropping the + session). +- **Codecs**: H.264 everywhere; HEVC/H.265 as an experimental opt-in + ("Prefer HEVC" in host Settings) with live mid-session switching. + +Not in scope yet: audio, USB transport, a first-party display driver (the +extended display uses the bundled MIT-licensed +[Virtual-Display-Driver](https://github.com/VirtualDrivers/Virtual-Display-Driver)), +and an App Store listing (the iPad app is TestFlight/Xcode-installed). + +## Install (testers) + +Grab **EternalMonitor-Setup.exe** from the +[latest release](https://github.com/whoisaldo/EternalMonitor/releases/latest), +run it, approve the one UAC prompt, and allow the firewall prompt (both +Private and Public). Step-by-step tester instructions live in +[scripts/QUICKSTART.txt](scripts/QUICKSTART.txt). The iPad app comes from +TestFlight (ask for an invite) or an Xcode build. + +SmartScreen note: the installer is not code-signed yet — "More info" → +"Run anyway". + +## Build from source + +The workspace is two Rust crates (`host/`, `proto/` = the pure `eternal-wire` +protocol crate) plus the Swift app in `ios/`. -- First-party signed display driver (today the host drives a third-party signed virtual display driver) -- USB transport -- Input relay back to Windows -- Reliability layer beyond basic UDP fragmentation/reassembly -- Production-grade zero-config discovery - -Known caveat: - -- The iPad "Scan network" path may fail to find the host even when direct IP connect works. Treat discovery as incomplete and use manual IP entry (or the QR code) when needed. -- NVIDIA (NVENC), AMD (AMF), and Intel (QSV) encode paths are implemented and hardened for the iPad VideoToolbox decoder; AMD is the current focus of beta testing. If a hardware encoder can't open, the host falls back to CPU (libx264) and shows a warning banner on the Stream tab. - -## How it works - -1. Windows host captures the desktop with DXGI Desktop Duplication -2. Frames are read back, converted, and encoded with hardware H.264 (auto-detected per GPU vendor, with NVIDIA currently the verified path) -3. Encoded `FramePacket` payloads are fragmented into UDP datagrams -4. The iPad app reassembles and parses those payloads -5. VideoToolbox decodes frames and Metal renders them - -Current target is practical local-network streaming, not a finished second-monitor product yet. +### Windows host -## Repo layout +Requirements: Rust stable (MSVC), an FFmpeg **7.1 shared** SDK, LLVM/libclang +(for bindgen). -```text -host/ Rust Windows host: capture, encode, UDP transport, mDNS advertisement, egui GUI -ios/ Swift iPad app: connect UI, UDP receive, reassembly, decode, render -proto/ Shared protocol serialization code and schemas -installer/ Inno Setup script + bundled-driver staging for EternalMonitor-Setup.exe -scripts/ package.ps1 (zip), build-installer.ps1 (Setup.exe), QUICKSTART.txt -docs/ eternalmonitor.dev website (GitHub Pages) +```powershell +# Point the build at your FFmpeg 7.1 shared SDK (folder containing bin\avcodec-*.dll) +$env:FFMPEG_DIR = "C:\ffmpeg" +cargo build --release -p eternal-host +.\target\release\eternal-host.exe # optional port argument, default 9876 ``` -## Build and run +`scripts\build-installer.ps1` builds the full Setup.exe (needs Inno Setup and +the same `FFMPEG_DIR`); `scripts\package.ps1` builds the bare zip. -Requires Windows with Rust stable MSVC and FFmpeg 7.1 for the host (GPU encoding auto-detected). The iOS app must be built on macOS with Xcode for a physical device. +### macOS development loop (no Windows required) -### Windows host +The host builds and runs on macOS with a synthetic capture source — the whole +protocol, encoder, transport, and supervisor stack is exercised for real: -```powershell -cargo build -p eternal-host -& "C:\Users\aliyo\OneDrive\Desktop\EternalMonitor\target\debug\eternal-host.exe" +```bash +brew install ffmpeg@7 pkgconf xcodegen +export PKG_CONFIG_PATH=/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig +cargo test --workspace # unit + golden-vector + synthetic end-to-end tests +ETERNAL_CAPTURE=synthetic cargo run -p eternal-host ``` -If you need to change the listen port: +If Xcode's command-line tools are the selected developer directory, prefix +Xcode commands with `DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer`. -```powershell -& "C:\Users\aliyo\OneDrive\Desktop\EternalMonitor\target\debug\eternal-host.exe" 9876 +### iPad app + +```bash +cd ios +xcodegen generate # project.yml is the source of truth +xcodebuild test -project EternalMonitor.xcodeproj -scheme EternalMonitor \ + -destination 'platform=iOS Simulator,name=iPad Pro 11-inch (M4)' ``` -### iOS app +Open the generated project in Xcode to run on a physical iPad (your own +signing team). -The iOS project is generated with XcodeGen: +### Full-system test on one Mac ```bash -cd ios -xcodegen generate +./scripts/e2e_ios.sh # host (synthetic) → iPad simulator, H.264 +EM_CODEC=hevc ./scripts/e2e_ios.sh # same, over HEVC ``` -Then open the generated Xcode project on macOS, build the `EternalMonitor` target, and run it on a physical iPad. +The harness launches the headless host and the simulator app, auto-connects, +and asserts ≥120 decoded frames at the right resolution via the app's +machine-readable log milestones. -## Installer (for testers) +## How it's tested -Non-technical testers don't need Rust or the zip. Build a single `EternalMonitor-Setup.exe`: +- **Golden wire vectors** (`proto/testdata/`) parsed byte-for-byte by both the + Rust and Swift codecs, plus fuzz "never crashes" tests on both sides. +- **Pure-logic unit tests** with injected clocks: session machine, ABR ladder, + pacer, reassembly, input mapping, gesture state machine, supervisor. +- **End-to-end tests** that run the real pipeline: Rust E2E (handshake, lossy + ABR step-down, encoder-crash recovery, input relay, HEVC negotiation) and + the simulator harness above. +- **CI** on every PR: Linux (wire crate), Windows (full host against pinned + FFmpeg 7.1), macOS (full workspace incl. E2E), and the iOS simulator suite. -```powershell -.\scripts\build-installer.ps1 -``` +What CI can't verify — real GPU encoders, the virtual display driver, real +WiFi — is covered by a hardware runbook before each release. + +## Repo layout -It compiles the release host, bundles the FFmpeg runtime, and — if a signed virtual -display driver is staged in `installer/vendor/vdd/` (see that folder's `README.txt`) — -bundles it too. The tester double-clicks the installer, approves one Windows (UAC) -prompt, and gets the app plus the virtual display installed in one run. The driver -install always requires that single elevation prompt; a fully seamless first-party -driver is a future (v0.2.0) goal. +```text +host/ Rust host: capture, encode, transport, session, supervisor, egui GUI +proto/ eternal-wire crate: protocol v2 codecs, H.264/HEVC helpers, golden vectors +ios/ Swift iPad app (xcodegen project): receive, decode, render, input relay +installer/ Inno Setup script + bundled-driver staging for EternalMonitor-Setup.exe +scripts/ build-installer.ps1, package.ps1, e2e_ios.sh, QUICKSTART.txt +docs/ eternalmonitor.dev website (GitHub Pages) +``` -Tester-facing instructions live in [scripts/QUICKSTART.txt](scripts/QUICKSTART.txt). +## Environment variables (host) + +| Variable | Effect | +| --- | --- | +| `ETERNAL_ENCODER` | Force an encoder (`h264_nvenc`, `h264_amf`, `h264_qsv`, `libx264`) | +| `ETERNAL_HEVC` | `1`/`0` overrides the HEVC preference (automation) | +| `ETERNAL_FPS` | Override target FPS | +| `ETERNAL_CAPTURE` | `synthetic` = generated test pattern instead of DXGI | +| `ETERNAL_HEADLESS` | `1` = run without the GUI until SIGTERM/SIGINT | +| `ETERNAL_VDD_TIMEOUT_SECS` | Virtual-display attach timeout | +| `ETERNAL_ABR` | `0` disables adaptive bitrate | +| `ETERNAL_DROP` | Test-only: inject fractional datagram loss | +| `ETERNAL_AMF_DIAG` | `1` = write AMF bitstream diagnostics | +| `ETERNAL_LEGACY_PTS` | `1` = old frame-counter PTS (escape hatch) | ## Troubleshooting -- If the iPad says no complete frame was reassembled, make sure both the Windows host and the iPad app were rebuilt from the same revision. The UDP fragment header changed in the working transport fix. -- If scan finds nothing, try direct IP connect first. Discovery failure does not necessarily mean streaming is broken. -- If you're testing on AMD, use the latest `v0.1.2-mirror` build. The AMF path prepends fresh SPS/PPS on every random-access frame (including forced non-IDR intra frames), recovers the startup keyframe if parameter sets aren't ready, and writes a first-120-packet capture to `%APPDATA%\EternalMonitor\diagnostics\` for offline inspection. -- If the host binary fails to rebuild on Windows with access denied for `eternal-host.exe`, close the running GUI process first. +- **iPad can't connect**: same WiFi (not a guest network), firewall allowed + for Private *and* Public, manual IP entry beats discovery on tricky + networks. The host window shows the address and a QR code. +- **Choppy video**: almost always WiFi. Get near the router, prefer 5 GHz, + wire the PC. The HUD's loss% and the host's ABR rung tell the story. +- **"H.264 (x264)" on the Stream tab**: the hardware encoder failed to open + and the host fell back to CPU encoding — update GPU drivers and restart the + stream. +- **Version mismatch**: protocol v2 is a clean break. A v0.1.x app or host + shows a clear "update the other side" message instead of streaming. ## Reference docs -- [ARCHITECTURE.md](ARCHITECTURE.md) -- [DECISIONS.md](DECISIONS.md) +- [ARCHITECTURE.md](ARCHITECTURE.md) — the pipeline, protocol v2, and design +- [DECISIONS.md](DECISIONS.md) — why things are the way they are - [RELEASE_NOTES.md](RELEASE_NOTES.md) - [FRIENDS_TESTING.md](FRIENDS_TESTING.md) — organizer notes for beta testing +- [HARDWARE_VERIFICATION.md](HARDWARE_VERIFICATION.md) — the pre-release + runbook for everything CI can't prove ## Credits diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d0c90a5..495aa58 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,58 @@ +## EternalMonitor v0.2.0 + +A ground-up revamp of the streaming core. **Clean break: the v0.2.0 host and +iPad app only work with each other** — each side shows a clear "update the +other half" message if it meets a v0.1.x peer. + +### Control the PC from the iPad +- Tap to click, drag to move the mouse, two-finger scroll, half-second hold + for a right-click, Apple Pencil with pressure. On by default ("Control PC + with touch" in the iPad Settings), negotiated per session — the host never + injects for a session that didn't ask. +- While control is on, a three-finger tap toggles the stats HUD. + +### Protocol v2 +- A real session: handshake with capability negotiation, busy rejection for a + second device, instant reconnect takeover, liveness tracking, and clean + goodbyes (including when the app is backgrounded). +- Host heartbeats, client receiver reports, keyframe requests, and NTP-style + clock sync — the HUD's latency number is now a real end-to-end measurement. +- Media is raw Annex B in a fixed 32-byte header; FlatBuffers is gone. + +### Reliability +- Adaptive bitrate: the host slider is now the **ceiling**; the stream steps + down under loss and back up when the network recovers. +- Keyframe recovery after loss (client-requested, host rate-limited), packet + pacing on keyframe bursts, and automatic reconnect with backoff after + "SIGNAL LOST". +- Host supervisor v2: an encoder crash auto-restarts the pipeline in about a + second and the iPad resumes on the same session — no reconnect, no + re-handshake. Wedge watchdogs catch silent stalls. + +### Video +- **HEVC/H.265** as an experimental opt-in ("Prefer HEVC" on the host): + negotiated per client, live mid-session codec switching, automatic H.264 + fallback. +- Real capture-time PTS (rate control finally sees true frame cadence), NV12 + decode output with proper BT.601/709 handling, aspect-fit rendering, and + draw-on-demand (no more free-running 120 Hz redraw). +- The extended (virtual) display now offers the **iPad's native resolution + and refresh rate**, and tears down when the client disconnects. + +### Quality of life +- Settings apply from the first frame (including headless runs), atomic + settings writes, mDNS advertisement without the periodic re-registration + gap plus goodbye packets on exit, live host info (name, resolution, codec, + bitrate) in the iPad Settings, "Keep screen awake" and "Resume after + switching apps" toggles, VoiceOver labels, and the Syne display font + actually rendering. +- The whole stack is now covered by tests: golden wire vectors parsed + byte-for-byte by both languages, pure-logic suites with injected clocks, + and end-to-end tests (including a full host→simulator stream in both + codecs) running in CI on Linux, Windows, and macOS. + +--- + ## EternalMonitor v0.1.2-mirror Reliability release focused on the AMD encode path and a seamless, on-demand extended display. diff --git a/docs/download.html b/docs/download.html index c5a0b70..3a26196 100644 --- a/docs/download.html +++ b/docs/download.html @@ -45,7 +45,7 @@

Download EternalMonitor

-

Windows host + iPad app · v0.1.2-mirror

+

Windows host + iPad app · matching versions required

@@ -58,7 +58,7 @@

Download EternalMonitor

Windows Host

- v0.1.2 + latest

Runs on your PC. Captures and streams your display.

@@ -100,12 +100,13 @@

iPad App

Requirements

  • Rust stable MSVC
  • -
  • FFmpeg 7.1
  • +
  • FFmpeg 7.1 shared SDK (set FFMPEG_DIR)
  • GPU with hardware encoder (NVIDIA/AMD/Intel) or software fallback

Windows Host

git clone https://github.com/whoisaldo/EternalMonitor
 cd EternalMonitor
+$env:FFMPEG_DIR = "C:\ffmpeg"   # your FFmpeg 7.1 shared SDK
 cargo build --release -p eternal-host

iPad App

cd ios
diff --git a/docs/index.html b/docs/index.html
index b42d12e..62be804 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -53,8 +53,8 @@ 

Your iPad.
A second monitor.

60fps - H.264 Hardware Encode - 120Hz ProMotion + Hardware H.264 / HEVC + Touch & Pencil control
@@ -63,8 +63,8 @@

Your iPad.
A second monitor.

- <20ms - latency + 2-way + video out, input back
60fps @@ -86,13 +86,13 @@

Your iPad.
A second monitor.

How it works

-

Three steps. No drivers to install, no account to create.

+

Three steps. One installer sets up everything — no account to create.

1

Run the host

-

Launch EternalMonitor on your Windows PC. It captures your display with DXGI and encodes with hardware H.264 (NVIDIA, AMD, or Intel).

+

Launch EternalMonitor on your Windows PC. It captures your display with DXGI and encodes on the GPU — H.264 by default, HEVC opt-in (NVIDIA, AMD, or Intel).

2
@@ -102,7 +102,7 @@

Connect your iPad

3

Start streaming

-

Your iPad is now a second display. Frames are streamed over UDP, decoded with VideoToolbox, and rendered with Metal.

+

Your iPad is now a second display you can touch: tap to click, drag, scroll, use the Pencil. The stream adapts its bitrate to your WiFi, recovers from loss, and shows its real measured latency in the HUD.

@@ -145,8 +145,8 @@

- v0.1.2 Preview -

v0.1.2 adds an on-demand extended display (via a bundled virtual display driver) and hardened multi-vendor encoding (NVIDIA/AMD/Intel). A first-party display driver, USB transport, and touch input are on the roadmap.

+ v0.2.0 +

v0.2.0 rebuilds the streaming core: a real session protocol, adaptive bitrate and loss recovery, touch/Pencil control of the PC, opt-in HEVC, and an extended display that matches the iPad's native resolution. Audio, USB transport, and a first-party display driver are on the roadmap.

@@ -165,17 +165,17 @@

Roadmap

Done
-
+
v0.2.0 - Extend + USB + Extend + Control
- Upcoming + Done
v0.3.0 - Touch input + Audio + USB
Upcoming
diff --git a/ios/project.yml b/ios/project.yml index de6c180..f8387fe 100644 --- a/ios/project.yml +++ b/ios/project.yml @@ -23,8 +23,8 @@ targets: PRODUCT_BUNDLE_IDENTIFIER: com.eternal.monitor TARGETED_DEVICE_FAMILY: "2" SUPPORTS_XR: "NO" - MARKETING_VERSION: "0.1.0" - CURRENT_PROJECT_VERSION: "3" + MARKETING_VERSION: "0.2.0" + CURRENT_PROJECT_VERSION: "5" DEVELOPMENT_TEAM: "9X79V37Q89" CODE_SIGN_STYLE: Automatic ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon diff --git a/scripts/QUICKSTART.txt b/scripts/QUICKSTART.txt index 1df3010..7fba62a 100644 --- a/scripts/QUICKSTART.txt +++ b/scripts/QUICKSTART.txt @@ -61,10 +61,23 @@ choose "Auto (primary)" and Restart stream again. The extra screen only exists while the iPad is connected and using it - when you quit EternalMonitor it disappears, so there's no leftover monitor. +Step 6 - Control the PC from the iPad +------------------------------------- +Your touches control the PC while streaming (on by default): + +- Tap = left click - Drag = move the mouse +- Two fingers = scroll - Hold ~0.5s = right click +- Apple Pencil = draws immediately - 3-finger tap = show/hide the stats + +To make the iPad view-only, turn off "Control PC with touch" in the iPad +app's Settings and reconnect. + If the video is choppy or laggy ------------------------------- - This is almost always Wi-Fi, not the app. Get close to the router, use - 5 GHz instead of 2.4 GHz, or plug the PC into Ethernet. + 5 GHz instead of 2.4 GHz, or plug the PC into Ethernet. The app now + lowers its quality automatically on a bad connection and sharpens back + up when it improves - brief softness is it coping, not breaking. - Open the "Performance" tab and check the Codec. If it reads "H.264 (x264)" you are encoding on the CPU (slow) because the hardware encoder didn't start - update your GPU drivers and click "Restart stream". diff --git a/scripts/build-installer.ps1 b/scripts/build-installer.ps1 index ede7971..77a14e6 100644 --- a/scripts/build-installer.ps1 +++ b/scripts/build-installer.ps1 @@ -8,11 +8,19 @@ # installer\vendor\vdd\ (see that folder's README.txt for the exact file). # Without it, the build still succeeds and produces an app-only installer. +param( + # Release builds (CI) pass this: an unsigned or invalidly-signed bundled + # driver then FAILS the build instead of warning. Local developer builds + # keep the warning so an unsigned test driver doesn't block iteration. + [switch]$StrictSignature +) + $ErrorActionPreference = "Stop" $repo = Split-Path -Parent $PSScriptRoot Set-Location $repo -$version = "0.1.2" +$version = [regex]::Match((Get-Content -Raw "host\Cargo.toml"), 'version\s*=\s*"([^"]+)"').Groups[1].Value +if (-not $version) { throw "Could not read the package version from host\Cargo.toml" } $staging = Join-Path $repo "build\installer-staging" $outDir = Join-Path $repo "build\out" @@ -33,7 +41,7 @@ Write-Host "[setup] ISCC: $iscc" # --- Build the release binary ---------------------------------------------------- Write-Host "[1/5] Building release binary..." -cargo build --release -p eternal-host +cargo build --release --locked -p eternal-host if ($LASTEXITCODE -ne 0) { throw "cargo build failed" } # --- Stage the payload ----------------------------------------------------------- @@ -44,14 +52,17 @@ New-Item -ItemType Directory -Path $outDir -Force | Out-Null Copy-Item "target\release\eternal-host.exe" (Join-Path $staging "EternalMonitor-host.exe") -# FFmpeg runtime DLLs + utility, resolved from .cargo/config.toml (same as package.ps1). -$cargoConfig = Get-Content -Raw ".cargo\config.toml" -$ffmpegMatch = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*\{\s*value\s*=\s*"([^"]+)"') -if (-not $ffmpegMatch.Success) { - $ffmpegMatch = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*"([^"]+)"') +# FFmpeg runtime DLLs + utility. FFMPEG_DIR (the same env var the build uses) +# wins; .cargo\config.toml is the fallback for developer machines that pin it there. +$ffmpegDir = $env:FFMPEG_DIR +if (-not $ffmpegDir -and (Test-Path ".cargo\config.toml")) { + $cargoConfig = Get-Content -Raw ".cargo\config.toml" + $m = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*\{\s*value\s*=\s*"([^"]+)"') + if (-not $m.Success) { $m = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*"([^"]+)"') } + if ($m.Success) { $ffmpegDir = $m.Groups[1].Value } } -if (-not $ffmpegMatch.Success) { throw "Could not resolve FFMPEG_DIR from .cargo/config.toml" } -$ffmpegBin = Join-Path $ffmpegMatch.Groups[1].Value "bin" +if (-not $ffmpegDir) { throw "FFMPEG_DIR is not set. Point it at your FFmpeg 7.1 shared SDK (the folder containing bin\avcodec-*.dll)." } +$ffmpegBin = Join-Path $ffmpegDir "bin" Copy-Item "$ffmpegBin\*.dll" $staging if (Test-Path "$ffmpegBin\ffmpeg.exe") { Copy-Item "$ffmpegBin\ffmpeg.exe" $staging } @@ -68,11 +79,18 @@ $includeDriver = $false if ($driverSetup) { $sig = Get-AuthenticodeSignature $driverSetup.FullName if ($sig.Status -ne "Valid") { + if ($StrictSignature) { + throw "Driver setup signature is '$($sig.Status)', expected 'Valid'. Refusing to bundle an unverified driver in a release build." + } Write-Warning "Driver setup signature is '$($sig.Status)', expected 'Valid'. Bundling anyway — verify the source." } $driverStage = Join-Path $staging "driver" New-Item -ItemType Directory -Path $driverStage | Out-Null Copy-Item $driverSetup.FullName (Join-Path $driverStage "vdd-setup-x64.exe") + # Redistribute the driver's license alongside it (it's a separate MIT + # project we bundle, not code we link). + Get-ChildItem $vddDir -Filter "LICENSE*" -ErrorAction SilentlyContinue | + ForEach-Object { Copy-Item $_.FullName $driverStage } $includeDriver = $true Write-Host " Bundling driver: $($driverSetup.Name) [signature: $($sig.Status)]" } else { diff --git a/scripts/package.ps1 b/scripts/package.ps1 index 7e985c2..ea31604 100644 --- a/scripts/package.ps1 +++ b/scripts/package.ps1 @@ -1,12 +1,32 @@ -# EternalMonitor release packaging script +# EternalMonitor release packaging script (zip, no installer) # Usage: .\scripts\package.ps1 +# +# FFmpeg runtime location: set the FFMPEG_DIR environment variable (the same +# one the build itself uses). If unset, falls back to .cargo\config.toml for +# developer machines that pin it there. -$version = "v0.1.2-mirror" +$ErrorActionPreference = "Stop" +$repo = Split-Path -Parent $PSScriptRoot +Set-Location $repo + +$version = [regex]::Match((Get-Content -Raw "host\Cargo.toml"), 'version\s*=\s*"([^"]+)"').Groups[1].Value +if (-not $version) { throw "Could not read the package version from host\Cargo.toml" } $distDir = "dist" -$zipName = "EternalMonitor-$version-windows.zip" +$zipName = "EternalMonitor-v$version-windows.zip" + +function Resolve-FfmpegDir { + if ($env:FFMPEG_DIR) { return $env:FFMPEG_DIR } + if (Test-Path ".cargo\config.toml") { + $cargoConfig = Get-Content -Raw ".cargo\config.toml" + $m = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*\{\s*value\s*=\s*"([^"]+)"') + if (-not $m.Success) { $m = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*"([^"]+)"') } + if ($m.Success) { return $m.Groups[1].Value } + } + throw "FFMPEG_DIR is not set. Point it at your FFmpeg 7.1 shared SDK (the folder containing bin\avcodec-*.dll)." +} Write-Host "[1/5] Building release binary..." -cargo build --release -p eternal-host +cargo build --release --locked -p eternal-host if ($LASTEXITCODE -ne 0) { exit 1 } Write-Host "[2/5] Preparing dist/..." @@ -15,15 +35,7 @@ New-Item -ItemType Directory -Path $distDir | Out-Null Write-Host "[3/5] Copying files..." Copy-Item "target\release\eternal-host.exe" "$distDir\EternalMonitor-host.exe" -$cargoConfig = Get-Content -Raw ".cargo\config.toml" -$ffmpegMatch = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*\{\s*value\s*=\s*"([^"]+)"') -if (-not $ffmpegMatch.Success) { - $ffmpegMatch = [regex]::Match($cargoConfig, 'FFMPEG_DIR\s*=\s*"([^"]+)"') -} -if (-not $ffmpegMatch.Success) { - throw "Could not resolve FFMPEG_DIR from .cargo/config.toml" -} -$ffmpegBin = (Join-Path $ffmpegMatch.Groups[1].Value "bin") +$ffmpegBin = Join-Path (Resolve-FfmpegDir) "bin" Copy-Item "$ffmpegBin\*.dll" "$distDir\" Copy-Item "$ffmpegBin\ffmpeg.exe" "$distDir\" Copy-Item "README.md" "$distDir\" From 8d993a7bcd3e0076f86e9441b67cde7e8816c1de Mon Sep 17 00:00:00 2001 From: whoisaldo Date: Wed, 26 Aug 2026 12:59:33 -0400 Subject: [PATCH 2/3] Docs prose pass Plainer sentences throughout the rewritten docs: dashes out, label-colon bullets converted, repeated pet words deduplicated. Historical release-notes sections keep their original text. Claude-Session: https://claude.ai/code/session_013ezpmTwAW6yAcRdy2DEaex --- ARCHITECTURE.md | 151 +++++++++++++------------- DECISIONS.md | 172 +++++++++++++++--------------- FRIENDS_TESTING.md | 49 ++++----- HARDWARE_VERIFICATION.md | 223 ++++++++++++++++++++------------------- README.md | 126 +++++++++++----------- RELEASE_NOTES.md | 20 ++-- 6 files changed, 383 insertions(+), 358 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cca7e9b..a1d7c97 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# EternalMonitor — Architecture +# EternalMonitor architecture Accurate as of v0.2.0 (protocol v2). @@ -23,103 +23,112 @@ Accurate as of v0.2.0 (protocol v2). supervised by supervisor.rs (health, watchdogs, backoff restarts) ``` -Host stages are dedicated OS threads connected by a **latest-wins frame slot** -(capture → encode: an unconsumed frame is displaced, so the encoder always -works on the freshest picture) and a **lossless channel** (encode → transport: -a dropped encoded P-frame would corrupt the GOP). Frame pixels travel in -`Arc>` buffers that are recycled — steady state does one full-frame -copy (the DXGI staging readback). - -The **supervisor** owns the pipeline: stage threads report their exit, wedge -watchdogs fire on silent stalls (loop heartbeat stale 3 s, no frame 5 s with a -client, encoder flat 3 s), and restarts get exponential backoff with a -restart-storm brake. The client **session lives outside the pipeline**, so a -crash-restart resumes streaming to the same session — the client sees a new -stream epoch and resets reassembly, with no re-handshake. +Host stages are dedicated OS threads. Capture hands frames to the encoder +through a latest-wins slot: an unconsumed frame gets displaced, so the +encoder always works on the freshest picture. The encoder hands access units +to transport through a lossless channel, because dropping an encoded P-frame +would corrupt the GOP. Frame pixels travel in `Arc>` buffers that +get recycled; steady state does one full-frame copy, the DXGI staging +readback. + +The supervisor owns the pipeline. Stage threads report their exits, wedge +watchdogs fire on silent stalls (loop heartbeat stale 3 s, no frame for 5 s +with a client connected, encoder flat 3 s), and restarts back off +exponentially with a restart-storm brake. The client session lives outside +the pipeline, so a crash-restart resumes streaming to the same session. The +client sees a new stream epoch, resets reassembly, and never re-handshakes. ## Wire protocol v2 One UDP socket carries both media and control. Every datagram starts with an 8-byte prefix: magic `"EM"`, version `2`, packet type, flags (media bit 0 = -keyframe), reserved, and a strict payload length. Legacy v1 datagrams began -with `"ET"`, so the two are unambiguous; v2 is otherwise a **clean break** -(each side tells the user to update the other on contact with v1). +keyframe), a reserved byte, and a strict payload length. Legacy v1 datagrams +began with `"ET"`, so the two never get confused. v2 is otherwise a clean +break; each side tells the user to update the other on contact with v1. -**Media** (type 0x01): a 32-byte header — session id, stream epoch, frame -sequence, fragment index/count (≤3066 ≈ 4 MiB per frame), capture timestamp -(µs on the host process clock) — followed by a raw Annex B chunk. No -serialization framework; width/height/codec travel in the control plane. +Media (type 0x01) is a 32-byte header followed by a raw Annex B chunk. The +header carries session id, stream epoch, frame sequence, fragment +index/count (up to 3066, about 4 MiB per frame), and the capture timestamp +in microseconds on the host process clock. There is no serialization +framework; width, height, and codec travel in the control plane. -**Control** (16-byte header: session id, message sequence, type): +Control messages share a 16-byte header (session id, message sequence, +type): | Message | Direction | Purpose | | --- | --- | --- | -| HELLO2 / HELLO_ACK | C→H / H→C | Session establishment: capability bits (H.264/HEVC decode, wants-input), screen size/refresh, nonce-idempotent ACK carrying session id, host-dictated timing, and the stream config | -| HEARTBEAT | H→C | 1 Hz liveness + embedded stream config (self-heals lost config changes) | -| RECEIVER_REPORT | C→H | 500 ms cadence: loss, completion, jitter, depths — feeds ABR and doubles as client liveness | -| KEYFRAME_REQUEST | C→H | Loss/decode-error recovery; host rate-limits to 1 per 500 ms | -| PING / PONG | C→H→C | NTP-style clock sync (min-RTT offset) for the honest end-to-end latency readout | -| STREAM_CONFIG | H→C | Immediate notify on bitrate/codec/resolution change | +| HELLO2 / HELLO_ACK | C→H / H→C | Session establishment. Capability bits (H.264/HEVC decode, wants-input), screen size and refresh, a nonce-idempotent ACK carrying the session id, host-dictated timing, and the stream config | +| HEARTBEAT | H→C | 1 Hz liveness plus the embedded stream config, which self-heals lost config changes | +| RECEIVER_REPORT | C→H | Every 500 ms: loss, completion, jitter, queue depths. Feeds ABR and doubles as client liveness | +| KEYFRAME_REQUEST | C→H | Recovery after loss or a decode error; the host rate-limits to one per 500 ms | +| PING / PONG | C→H→C | NTP-style clock sync (min-RTT offset) behind the HUD's measured end-to-end latency | +| STREAM_CONFIG | H→C | Immediate notify on a bitrate, codec, or resolution change | | INPUT_EVENT | C→H | Input relay (below) | | BYE | both | Clean teardown with a reason (user, backgrounded, shutdown) | -**Session rules** (host, pure state machine): one client at a time — a second -device gets `busy`; the same device reconnecting supersedes in place with a -fresh session id; duplicate HELLO2 nonces get an identical ACK (retransmit -tolerance); liveness expires 3 s after the last report/input, which also -tears down the virtual display. +Session rules, implemented as a pure state machine on the host: one client +at a time, and a second device gets `busy`. The same device reconnecting +supersedes in place with a fresh session id. Duplicate HELLO2 nonces get an +identical ACK, which makes handshake retransmits harmless. Liveness expires +3 s after the last report or input event, and expiry also tears down the +virtual display. ## Reliability -- **ABR**: a bitrate ladder (4–20 Mbps, capped by the GUI "Max bitrate" - slider) driven by receiver reports — loss or PLI pressure steps down - (cooldown 3 s), 15 s of clean reports steps back up. A bitrate change - reopens the encoder session (~50–200 ms, same epoch) and forces an IDR. -- **Pacing**: keyframe bursts are sent in batches with microsleeps under a - hard 3 ms wall-clock budget, plus a 4 MiB kernel send buffer — smoothing - WiFi loss without adding latency. -- **Recovery**: the client requests keyframes on gaps/decode errors; the host - honors at most 2/s. The app shows "SIGNAL LOST" after 3 s of silence and - auto-reconnects with backoff. +- ABR: a bitrate ladder (4 to 20 Mbps, capped by the GUI "Max bitrate" + slider) driven by receiver reports. Loss or keyframe-request pressure + steps down with a 3 s cooldown; 15 s of clean reports steps back up. A + bitrate change reopens the encoder session (50 to 200 ms, same epoch) and + forces an IDR. +- Pacing: keyframe bursts go out in batches with microsleeps under a hard + 3 ms wall-clock budget, on a socket with a 4 MiB kernel send buffer. This + smooths WiFi loss without adding latency. +- Recovery: the client requests keyframes on gaps and decode errors; the + host honors at most two per second. The app shows "SIGNAL LOST" after 3 s + of silence and reconnects with backoff. ## Input relay -The iPad normalizes touches over the displayed video (letterbox-corrected) to -a 0–65535 grid. A pure gesture machine turns raw touches into events: tap = -click at the touch-down point, drag commits after slop, two fingers scroll -(direct-manipulation direction), a 500 ms hold right-clicks, Apple Pencil -presses immediately with pressure. Press/release edges are sent twice with one -event id; the host dedupes, maps through the captured output's desktop -rectangle onto the Windows virtual screen, and injects with `SendInput`. -Sessions that didn't set the wants-input capability are never injected for. +The iPad normalizes touches over the displayed video (letterbox-corrected) +to a 0–65535 grid. A pure gesture machine turns raw touches into events: tap +means click at the touch-down point, a drag commits after slop, two fingers +scroll in the direct-manipulation direction, a 500 ms hold right-clicks, and +Apple Pencil presses immediately with pressure. Press/release edges are sent +twice with one event id; the host dedupes, maps through the captured +output's desktop rectangle onto the Windows virtual screen, and injects with +`SendInput`. Sessions that didn't set the wants-input capability never get +injected for. ## Codecs -H.264 is the default everywhere. With the host's "Prefer HEVC" setting on and -a client that advertises HEVC decode, the encoder live-switches to the HEVC -sibling (NVENC/AMF/QSV/VideoToolbox/x265) via the same reopen mechanism. The -iPad decoder detects the codec **from the bitstream** (an HEVC VPS in a -keyframe), so the switch has no config/media ordering race. HEVC encoders are -configured to repeat VPS/SPS/PPS in-band; H.264 keyframes get cached SPS/PPS -prepended (the AMF path has extra guards developed against real hardware). +H.264 is the default everywhere. With the host's "Prefer HEVC" setting on +and a client that advertises HEVC decode, the encoder live-switches to the +HEVC sibling (NVENC/AMF/QSV/VideoToolbox/x265) through the same reopen +mechanism. The iPad decoder detects the codec from the bitstream itself (an +HEVC VPS in a keyframe), so the switch has no ordering race between config +and media. HEVC encoders repeat VPS/SPS/PPS in-band; H.264 keyframes get +cached SPS/PPS prepended. The AMF path carries extra guards developed +against real hardware. ## Discovery The host advertises `_eternaldisplay._udp` over mDNS with `version`, -`proto=2`, and `platform` TXT records, re-upserted every 60 s (no -unregister gap) and withdrawn with goodbye packets on exit. Manual IP entry -and the QR code remain the fallback for networks that filter multicast. +`proto=2`, and `platform` TXT records. The advertisement re-upserts every +60 s without an unregister gap and goes out with goodbye packets on exit. +Manual IP entry and the QR code remain the fallback for networks that +filter multicast. ## Portability and testing Everything protocol- or logic-shaped lives in the pure `eternal-wire` crate -(v2 codecs, H.264/HEVC bitstream helpers, reassembly) or in host modules with -injected clocks (session, ABR, pacer, supervisor, input mapping) — all tested -on every platform. Windows-only code (DXGI, SendInput, VDD control) is -`cfg(windows)`-gated; a synthetic capture source with a machine-readable -frame counter lets the full host run on macOS/CI, where the Rust E2E suite -drives a fake receiver through handshake, loss, crash-recovery, input, and -HEVC scenarios, and `scripts/e2e_ios.sh` streams into the real iPad app in -the simulator. Golden wire vectors are parsed byte-for-byte by both -languages. What only hardware can prove (GPU encoders, the display driver, -real WiFi) is enumerated in the release hardware runbook. +(v2 codecs, H.264/HEVC bitstream helpers, reassembly) or in host modules +with injected clocks (session, ABR, pacer, supervisor, input mapping), all +tested on every platform. Windows-only code (DXGI, SendInput, VDD control) +sits behind `cfg(windows)`. A synthetic capture source with a +machine-readable frame counter lets the full host run on macOS and CI, +where the Rust E2E suite drives a fake receiver through handshake, loss, +crash-recovery, input, and HEVC scenarios, and `scripts/e2e_ios.sh` streams +into the real iPad app in the simulator. Golden wire vectors are parsed +byte-for-byte by both languages. What only hardware can prove (GPU +encoders, the display driver, real WiFi) is enumerated in +HARDWARE_VERIFICATION.md. diff --git a/DECISIONS.md b/DECISIONS.md index bced713..9103d21 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -1,4 +1,4 @@ -# EternalMonitor — Technical Decisions +# EternalMonitor technical decisions ## Language choices @@ -8,100 +8,103 @@ - Works well with `tokio`, `windows`, and `ffmpeg-next` - Keeps the host codebase native and low-level without switching to C++ -Rejected: C++ (more binding/build overhead for this repo shape), C# (wrong -fit for the native graphics + transport path). +Rejected: C++ (more binding and build overhead for this repo shape), C# +(wrong fit for the native graphics and transport path). ### Swift for the iPad app -- Direct access to VideoToolbox, Metal, Network, and UIKit/SwiftUI APIs +Direct access to VideoToolbox, Metal, Network, and UIKit/SwiftUI. -Rejected: React Native / Flutter — wrong fit for this decode/render stack. +Rejected: React Native / Flutter. Wrong fit for this decode/render stack. ## Capture ### DXGI Desktop Duplication, CPU readback The practical Windows desktop-capture API. The pipeline does one full-frame -copy (staging-texture readback into a recycled `Arc` buffer); cursor -compositing is CPU-side. Dirty-rect metadata is queried but not yet used for +copy (staging-texture readback into a recycled `Arc` buffer) and composites +the cursor on the CPU. Dirty-rect metadata is queried but not yet used for partial encode. A zero-copy GPU path (capture texture straight into the -encoder) remains future work — measure first: after the v0.2.0 hot-path work, -the readback is no longer the dominant cost at 1080p60. +encoder) remains future work; measure first, because after the v0.2.0 +hot-path work the readback is no longer the dominant cost at 1080p60. ## Encoder ### `ffmpeg-next` pinned to FFmpeg 7.1, multi-vendor hardware encode -The host detects the GPU vendor (DXGI adapter enumeration) and walks a -vendor-preferred chain — NVENC / AMF / QSV, then libx264 — with per-encoder -low-latency options. FFmpeg is pinned to **7.1** everywhere (Windows DLLs, -CI's downloaded SDK, Homebrew `ffmpeg@7`) so the bytes tested are the bytes -shipped; upgrading to 8.x is deliberately a separate change now that the E2E +The host detects the GPU vendor via DXGI adapter enumeration and walks a +vendor-preferred chain (NVENC / AMF / QSV, then libx264) with per-encoder +low-latency options. FFmpeg is pinned to 7.1 everywhere: the Windows DLLs, +CI's downloaded SDK, and Homebrew `ffmpeg@7`. The bytes tested are the bytes +shipped. Upgrading to 8.x is deliberately a separate change now that the E2E harness exists to validate it. AMF is the fragile path (startup keyframes without parameter sets, strict -VideoToolbox level requirements) and carries bespoke guards developed against -real hardware; those guards are preserved verbatim through refactors. +VideoToolbox level requirements). Its guards were developed against real +hardware and get preserved verbatim through refactors. ### H.264 baseline default, HEVC opt-in -Baseline H.264 is the simplest compatibility target for VideoToolbox startup. -HEVC ships behind a host setting ("Prefer HEVC") until it has been proven on -each hardware encoder — negotiation requires both the setting and the -client's advertised decode capability, and any HEVC open failure falls back -to H.264 silently. +Baseline H.264 is the simplest compatibility target for VideoToolbox +startup. HEVC ships behind a host setting ("Prefer HEVC") until it has been +proven on each hardware encoder. Negotiation requires both the setting and +the client's advertised decode capability, and any HEVC open failure falls +back to H.264 silently. ### Encoder reconfiguration = session reopen Hardware encoders ignore bitrate pokes on an open context, so every real -change (ABR rung, slider, codec switch) reopens the encoder session (~50–200 -ms, same stream epoch) and forces an IDR. The old per-frame `apply_bitrate` -call — which silently did nothing on NVENC/AMF — is gone; the GUI value is -now the ABR **ceiling**, labeled accordingly. +change (ABR rung, slider, codec switch) reopens the encoder session (50 to +200 ms, same stream epoch) and forces an IDR. The old per-frame +`apply_bitrate` call, which silently did nothing on NVENC and AMF, is gone. +The GUI value is now the ABR ceiling and is labeled accordingly. ## Transport & protocol -### Protocol v2: raw Annex B over a custom UDP framing, one socket for media + control +### Protocol v2: raw Annex B over custom UDP framing, one socket for media and control -v2 replaced the v1 format (FlatBuffers `FramePacket` + 16-byte fragment -header + fire-and-forget `ETERNALHELLO`) as a **clean break** — both sides +v2 replaced the v1 format (FlatBuffers `FramePacket`, a 16-byte fragment +header, and a fire-and-forget `ETERNALHELLO`) as a clean break. Both sides ship together, and each side recognizes the other's legacy traffic well -enough to say "update the other half". Rationale for the break: the v1 -format had no version field, no session identity, and no back channel, so -compatible evolution wasn't possible. - -- **FlatBuffers removed**: the only dynamic field was the frame payload - itself; width/height/codec belong to the control plane (STREAM_CONFIG), - so media is now a fixed 32-byte header + raw Annex B. One less - serialization layer on the per-frame hot path, one less parser to fuzz. -- **No app-layer checksum/CRC**: UDP's checksum plus magic/version/session - checks and strict length validation catch stray and truncated datagrams; - a corrupted-but-valid datagram costs at most an artifact until the next - keyframe, which the PLI path requests anyway. A CRC would tax every packet - to protect against the rarest failure with the mildest consequence. -- **FEC deferred**: consumer-WiFi loss is bursty, which single-XOR parity - handles poorly. Pacing + keyframe recovery + ABR carry v0.2.0; packet - types and flag bits are reserved for FEC if measurement ever justifies it. -- **Client liveness = its receiver reports** (500 ms cadence) rather than a - dedicated client heartbeat — the reports must flow anyway to drive ABR. - -Rejected: TCP (head-of-line blocking), WebRTC (too heavy), RTP/RTCP (we'd -use a fraction of it and still need custom extensions for input/config). +enough to say "update the other half". The break was the point: v1 had no +version field, no session identity, and no back channel, so compatible +evolution wasn't possible. + +- FlatBuffers removed. The only dynamic field was the frame payload itself; + width, height, and codec belong in the control plane (STREAM_CONFIG). So + media is a fixed 32-byte header plus raw Annex B. One less serialization + layer on the per-frame hot path, one less parser to fuzz. +- No app-layer checksum. UDP's checksum plus magic/version/session checks + and strict length validation catch stray and truncated datagrams. A + corrupted-but-valid datagram costs at most an artifact until the next + keyframe, which the recovery path requests anyway. A CRC would tax every + packet to protect against the rarest failure with the mildest + consequence. +- FEC deferred. Consumer-WiFi loss is bursty, which single-XOR parity + handles poorly. Pacing, keyframe recovery, and ABR carry v0.2.0; packet + types and flag bits stay reserved for FEC if measurement ever justifies + it. +- Client liveness is its receiver reports (500 ms cadence) rather than a + dedicated heartbeat message. The reports must flow anyway to drive ABR. + +Rejected: TCP (head-of-line blocking), WebRTC (too heavy), RTP/RTCP (we +would use a fraction of it and still need custom extensions for input and +config). ### Adaptive bitrate on the host, signals from the client -The host owns the ladder (it owns the encoder); the client just reports -honestly. Loss or keyframe-request pressure steps down, sustained clean -reports step up, and the GUI slider caps the ladder. +The host owns the ladder because it owns the encoder; the client just +reports what it sees. Loss or keyframe-request pressure steps down, +sustained clean reports step up, and the GUI slider caps the ladder. ## Discovery ### mDNS/DNS-SD, best effort, manual IP as the guaranteed path The host advertises `_eternaldisplay._udp` (TXT: `version`, `proto=2`, -`platform`), re-upserting every 60 s — the v0.1.x unregister-then-register -refresh created a periodic discovery hole — and sends goodbye packets on -exit. Multicast-filtering networks still exist, so manual IP + QR remain +`platform`), re-upserting every 60 s. The v0.1.x refresh unregistered +first, which created a periodic discovery hole. Exit sends goodbye packets. +Multicast-filtering networks still exist, so manual IP and the QR code stay first-class. ## Virtual display @@ -109,45 +112,46 @@ first-class. ### Bundled third-party Indirect Display Driver, managed on demand The extended display drives the bundled -[VirtualDrivers/Virtual-Display-Driver](https://github.com/VirtualDrivers/Virtual-Display-Driver) -(**MIT licensed — verified; its license text ships in the installer next to -the driver**). The host doesn't run elevated, so the installer registers two -SYSTEM scheduled tasks (enable/disable) the host triggers via `schtasks` — -no per-toggle UAC. - -- Enabled **only while an iPad is connected**; disabled on exit, target - change, startup, panic, and now on client loss (the v2 liveness signal - finally made idle teardown possible). -- Before enabling, the host writes `vdd_settings.xml` so the virtual display - offers the iPad's native landscape resolution and refresh (opt-out in - Settings). +[VirtualDrivers/Virtual-Display-Driver](https://github.com/VirtualDrivers/Virtual-Display-Driver). +Its MIT license is verified, and its license text ships in the installer +next to the driver. The host doesn't run elevated, so the installer +registers two SYSTEM scheduled tasks (enable/disable) that the host +triggers via `schtasks`, avoiding a per-toggle UAC prompt. + +- Enabled only while an iPad is connected. Disabled on exit, target change, + startup, panic, and on client loss (the v2 liveness signal finally made + idle teardown possible). +- Before enabling, the host writes `vdd_settings.xml` so the virtual + display offers the iPad's native landscape resolution and refresh. There + is an opt-out in Settings. - A first-party signed display driver remains a long-term goal. ## Input relay -### Wire normalized coordinates; a pure gesture machine on the client; `SendInput` on the host +### Normalized coordinates on the wire, a pure gesture machine on the client, `SendInput` on the host -Touches are normalized over the *displayed video* (letterbox-corrected), so -the wire format is resolution-independent. Tap-vs-drag-vs-scroll -disambiguation is a pure state machine (tap = click on release; drag commits -after slop; pencil presses immediately because ink can't wait). Edges are -sent twice with one event id and deduped host-side — loss tolerance without -retransmit machinery. Injection maps through the captured output's desktop -rectangle, so multi-monitor and virtual-display layouts land clicks on the -right screen. +Touches are normalized over the displayed video (letterbox-corrected), which +makes the wire format resolution-independent. Tap-vs-drag-vs-scroll +disambiguation is a pure state machine: tap clicks on release, a drag +commits after slop, and the Pencil presses immediately because ink can't +wait out a disambiguation window. Edges are sent twice with one event id and +deduped host-side, which buys loss tolerance without retransmit machinery. +Injection maps through the captured output's desktop rectangle, so +multi-monitor and virtual-display layouts land clicks on the right screen. ## Versioning & release -- One version, single-sourced from `host/Cargo.toml` (`env!` into the banner - and the mDNS TXT), matched by the iOS `MARKETING_VERSION`. -- Releases are built by CI from a `v*` tag: pinned FFmpeg, pinned driver - version, **hard-fail Authenticode verification** on the bundled driver, - and the installer's SHA-256 published in the release body (the website - reads it from there). +- One version, single-sourced from `host/Cargo.toml` (`env!` into the + banner and the mDNS TXT), matched by the iOS `MARKETING_VERSION`. +- CI builds releases from a `v*` tag: pinned FFmpeg, pinned driver version, + hard-fail Authenticode verification on the bundled driver, and the + installer's SHA-256 published in the release body, where the website + reads it. ## Deferred -- Audio (WASAPI → Opus → AVAudioEngine sketch exists; wire types reserved) +- Audio (a WASAPI → Opus → AVAudioEngine sketch exists; wire types are + reserved) - USB transport - First-party signed display driver -- FEC (types/flags reserved), zero-copy GPU capture, dirty-rect encode +- FEC (types and flags reserved), zero-copy GPU capture, dirty-rect encode diff --git a/FRIENDS_TESTING.md b/FRIENDS_TESTING.md index 3a9dc0d..170cd08 100644 --- a/FRIENDS_TESTING.md +++ b/FRIENDS_TESTING.md @@ -28,27 +28,27 @@ so its display task is registered. The host and iPad app must be from the **same release**. v0.2.0's protocol v2 is a deliberate clean break: a v0.1.x app meeting a v0.2.0 host (or the -reverse) won't stream — each side shows an explicit "update the other half" +reverse) won't stream. Each side shows an explicit "update the other half" message instead of corrupted video, so at least the failure is obvious. Hand out the matching TestFlight build and installer together. -## New in v0.2.0 — things worth testing on purpose - -- **Input relay**: tap/drag/two-finger scroll/hold-for-right-click/Pencil - should feel like a trackpad. Check multi-monitor setups (clicks must land - on the captured screen) and the "Control PC with touch" toggle off (host - must ignore touches). -- **HEVC**: flip "Prefer HEVC" in host Settings mid-stream; the codec on the - iPad's Settings HOST module should flip to HEVC within a second, and back. - If video breaks only in HEVC on some GPU, that encoder's HEVC path is the - bug — collect logs and turn the toggle off. -- **Recovery**: kill the host mid-stream (Task Manager) and relaunch — the - iPad should show SIGNAL LOST and reconnect by itself. Walk to the edge of - WiFi range — the picture should coarsen (bitrate stepping down) rather - than freeze, and recover afterwards. -- **Extended display resolution**: with "Match extended display to the - iPad's resolution" on (default), the virtual display should come up at the - iPad's native aspect — no letterboxing on the iPad. +## New in v0.2.0, worth testing on purpose + +- Input relay: tap, drag, two-finger scroll, hold for right-click, and the + Pencil should feel like a trackpad. Check multi-monitor setups (clicks + must land on the captured screen) and the "Control PC with touch" toggle + off (the host must ignore touches). +- HEVC: flip "Prefer HEVC" in host Settings mid-stream. The codec in the + iPad's Settings HOST module should flip to HEVC within a second, and + back. If video breaks only in HEVC on some GPU, that encoder's HEVC path + is the bug; collect logs and turn the toggle off. +- Recovery: kill the host mid-stream (Task Manager) and relaunch. The iPad + should show SIGNAL LOST and reconnect by itself. Walk to the edge of WiFi + range; the picture should coarsen (bitrate stepping down) rather than + freeze, and recover afterwards. +- Extended display resolution: with "Match extended display to the iPad's + resolution" on (the default), the virtual display should come up at the + iPad's native aspect, with no letterboxing on the iPad. ## Cover all three GPU vendors @@ -90,9 +90,10 @@ now shows an **amber warning banner** on the Stream tab. CPU encoding is hot and ## Known limitations (so you don't chase ghosts) -No audio, no USB transport, and no retransmit of lost packets — loss shows as -a brief artifact or frame skip, then the stream self-heals with a requested -keyframe (and steps the bitrate down if loss persists). Sustained stutter on -a clean network IS reportable now; on hotel/guest WiFi it's still the -network. HEVC is experimental and off by default — if a stream misbehaves, -confirm the codec on the Stream tab before filing it as a general bug. +No audio, no USB transport, and no retransmit of lost packets. Loss shows +as a brief artifact or frame skip, then the stream self-heals with a +requested keyframe and steps the bitrate down if loss persists. Sustained +stutter on a clean network IS reportable now; on hotel or guest WiFi it's +still the network. HEVC is experimental and off by default, so if a stream +misbehaves, confirm the codec on the Stream tab before filing it as a +general bug. diff --git a/HARDWARE_VERIFICATION.md b/HARDWARE_VERIFICATION.md index 9040113..f2b52d0 100644 --- a/HARDWARE_VERIFICATION.md +++ b/HARDWARE_VERIFICATION.md @@ -1,146 +1,153 @@ # v0.2.0 Hardware Verification Runbook What CI cannot prove: real GPU encoders, the virtual display driver, real -WiFi, and touch feel. Run this on the Windows PC + a real iPad before -tagging the release. Budget ~45 minutes (plus per-GPU repeats if you can -borrow AMD/Intel machines). Check items off; anything that fails gets logs -(see the last section) and a fix round before the tag. +WiFi, and touch feel. Run this on the Windows PC plus a real iPad before +tagging the release. Budget about 45 minutes, plus per-GPU repeats if you +can borrow AMD or Intel machines. Check items off; anything that fails gets +logs (see the last section) and a fix round before the tag. -Conventions: **Expect** is the pass condition. `Host log` = Copy logs button -(Stream tab) or `%APPDATA%\EternalMonitor\logs\eternal-host-session.log`. +Conventions: "Expect" is the pass condition. "Host log" means the Copy logs +button on the Stream tab, or +`%APPDATA%\EternalMonitor\logs\eternal-host-session.log`. ## A. Install & first light (5 min) -- [ ] **A1 — Installer**: run `EternalMonitor-Setup.exe` (SmartScreen → "Run - anyway", one UAC prompt). *Expect*: install completes, host launches, no +- [ ] **A1 Installer.** Run `EternalMonitor-Setup.exe` (SmartScreen → "Run + anyway", one UAC prompt). Expect: install completes, host launches, no second UAC. Upgrading over a previous version keeps settings. -- [ ] **A2 — Firewall**: on first run tick BOTH Private and Public. - *Expect*: prompt appears exactly once. -- [ ] **A3 — Banner**: host log shows `EternalMonitor v0.2.0`, the right GPU - name, and a hardware encoder (not x264) with no fallback banner in the GUI. +- [ ] **A2 Firewall.** On first run tick BOTH Private and Public. Expect: + the prompt appears exactly once. +- [ ] **A3 Banner.** Host log shows `EternalMonitor v0.2.0`, the right GPU + name, and a hardware encoder (not x264), with no fallback banner in the + GUI. ## B. Basic streaming (8 min) -- [ ] **B1 — Manual IP connect**: iPad → enter the host IP. *Expect*: - picture in under 2 s, iPad HUD ~60 fps, host Stream tab shows the client. -- [ ] **B2 — Discovery**: iPad Scan finds the host. Leave the scan list open - 4+ minutes (crosses two 60 s re-advertisements). *Expect*: the host never - blinks out of the list. Quit the host app. *Expect*: it leaves the list - within a few seconds (mDNS goodbye), not after minutes. -- [ ] **B3 — QR connect**: scan the host's QR from the iPad. *Expect*: +- [ ] **B1 Manual IP connect.** Enter the host IP on the iPad. Expect: a + picture in under 2 s, iPad HUD around 60 fps, host Stream tab shows the + client. +- [ ] **B2 Discovery.** The iPad Scan list finds the host. Leave the list + open 4+ minutes, which crosses two 60 s re-advertisements. Expect: the + host never blinks out of the list. Quit the host app. Expect: it leaves + the list within a few seconds (mDNS goodbye), not after minutes. +- [ ] **B3 QR connect.** Scan the host's QR from the iPad. Expect: it connects to the same address the GUI shows. -- [ ] **B4 — Truthful readouts**: host Stream tab codec matches the iPad - Settings → HOST module (name, resolution, fps, codec, bitrate), and the - HOST bitrate follows the ABR rung, not just the slider. -- [ ] **B5 — Latency sanity**: drag a window in circles; the iPad HUD's ms - readout should sit in the tens (typically 20–80 ms on good WiFi) and the - motion should feel attached. If you have a 240 fps camera, film both - screens and count frames — HUD claim within ~±20 ms of measured. -- [ ] **B6 — Decoder**: iPad Settings diagnostics say "hardware decoder" - (the simulator's software path must not appear on device). +- [ ] **B4 Truthful readouts.** The host Stream tab codec matches the iPad + Settings HOST module (name, resolution, fps, codec, bitrate), and the + HOST bitrate follows the adaptive rung, not just the slider. +- [ ] **B5 Latency sanity.** Drag a window in circles. The iPad HUD's ms + readout should sit in the tens (typically 20 to 80 ms on good WiFi) and + the motion should feel attached. With a 240 fps camera, film both screens + and count frames; the HUD claim should land within about ±20 ms of + measured. +- [ ] **B6 Decoder.** iPad Settings diagnostics say "hardware decoder". The + simulator's software path must not appear on a real device. ## C. Input relay (7 min) -- [ ] **C1 — Click targets**: tap small targets (window close buttons) in - all four screen corners. *Expect*: exact hits — no offset (this validates - desktop-rect mapping; test at 100% AND at 150% display scaling). -- [ ] **C2 — Drag**: drag a window smoothly; text selection works; no - spurious clicks when starting a two-finger scroll. -- [ ] **C3 — Scroll**: two-finger scroll in a browser. *Expect*: content +- [ ] **C1 Click targets.** Tap small targets (window close buttons) in all + four screen corners. Expect: exact hits with no offset. This validates + the desktop-rect mapping; test at 100% AND at 150% display scaling. +- [ ] **C2 Drag.** Drag a window smoothly; select text; starting a + two-finger scroll must not produce a stray click. +- [ ] **C3 Scroll.** Two-finger scroll in a browser. Expect: content follows the fingers (direct-manipulation direction), smooth, both axes. -- [ ] **C4 — Right-click**: hold ~½ s. *Expect*: context menu at the touch - point. Tap elsewhere dismisses it (single click, not double). -- [ ] **C5 — Pencil**: in Paint/whiteboard, ink starts immediately on - contact (no tap-vs-drag delay) and pressure varies the stroke where - supported. -- [ ] **C6 — Multi-monitor**: with a second physical monitor attached, - capture monitor 2 — touches must land on monitor 2, never the primary. -- [ ] **C7 — View-only**: turn "Control PC with touch" off, reconnect. - *Expect*: touches do nothing on the PC; single-tap toggles the HUD. +- [ ] **C4 Right-click.** Hold about half a second. Expect: a context menu + at the touch point. A tap elsewhere dismisses it with a single click, not + a double. +- [ ] **C5 Pencil.** In Paint or a whiteboard, ink starts immediately on + contact (no tap-vs-drag delay) and pressure varies the stroke where the + app supports it. +- [ ] **C6 Multi-monitor.** With a second physical monitor attached, + capture monitor 2. Touches must land on monitor 2, never the primary. +- [ ] **C7 View-only.** Turn "Control PC with touch" off and reconnect. + Expect: touches do nothing on the PC, and a single tap toggles the HUD. ## D. Reliability (8 min) -- [ ] **D1 — Host death**: kill the host from Task Manager mid-stream. - *Expect*: iPad shows SIGNAL LOST within ~3 s. Relaunch the host. - *Expect*: the iPad reconnects by itself (no taps) within ~10 s. -- [ ] **D2 — ABR under real loss**: walk toward the edge of WiFi range (or - microwave the link). *Expect*: picture softens (host GUI bitrate steps - down), no multi-second freezes; walking back sharpens it within ~20 s. -- [ ] **D3 — Live bitrate change**: move the Max-bitrate slider mid-stream. - *Expect*: a sub-second hiccup at most, no disconnect, no epoch weirdness. -- [ ] **D4 — Backgrounding**: swipe the app away to the switcher. *Expect*: - host Stream tab returns to "waiting for client" within ~3 s (BYE), and if - streaming the virtual display it tears down. Reopen the app. *Expect*: - auto-resume ("Resume after switching apps" default on). -- [ ] **D5 — Second device busy**: while one iPad streams, connect from a - second (or the simulator). *Expect*: clear "host is busy" message; the - first stream is untouched. -- [ ] **D6 — Version mismatch UX** (if a v0.1.x build is still around): - old app → new host and new app → old host each show an explicit "update - the other side" message, not garbage video. +- [ ] **D1 Host death.** Kill the host from Task Manager mid-stream. + Expect: the iPad shows SIGNAL LOST within about 3 s. Relaunch the host. + Expect: the iPad reconnects by itself within about 10 s, no taps. +- [ ] **D2 ABR under real loss.** Walk toward the edge of WiFi range (or + run the microwave). Expect: the picture softens as the host bitrate steps + down, with no multi-second freezes; walking back sharpens it within about + 20 s. +- [ ] **D3 Live bitrate change.** Move the Max-bitrate slider mid-stream. + Expect: at most a sub-second hiccup, no disconnect. +- [ ] **D4 Backgrounding.** Swipe the app away to the switcher. Expect: the + host Stream tab returns to "waiting for client" within about 3 s (BYE), + and a virtual display tears down. Reopen the app. Expect: it resumes by + itself ("Resume after switching apps" defaults on). +- [ ] **D5 Second device busy.** While one iPad streams, connect from a + second device. Expect: a clear "host is busy" message, and the first + stream is untouched. +- [ ] **D6 Version mismatch UX** (if a v0.1.x build is still around). Old + app → new host and new app → old host each show an explicit "update the + other side" message, not garbage video. ## E. HEVC (5 min, repeat per GPU vendor available) -- [ ] **E1 — Switch on**: mid-stream, tick "Prefer HEVC". *Expect*: iPad - HOST module codec flips to HEVC within ~1 s, picture stays clean, no - reconnect. Untick → back to H.264 the same way. -- [ ] **E2 — Quality/limits**: at the same bitrate HEVC should look no worse - than H.264. Watch 2+ minutes for artifacts, especially on **AMF** (its - `header_insertion_mode` handling is the least-proven path). Any breakage: - note GPU + driver version, collect logs, and leave the toggle off. -- [ ] **E3 — Fallback**: on a GPU without an HEVC encoder, the toggle warns - once in the log and keeps streaming H.264 (no error loop). +- [ ] **E1 Switch on.** Mid-stream, tick "Prefer HEVC". Expect: the iPad + HOST module codec flips to HEVC within about 1 s, the picture stays + clean, no reconnect. Untick and it returns to H.264 the same way. +- [ ] **E2 Quality and limits.** At the same bitrate HEVC should look no + worse than H.264. Watch 2+ minutes for artifacts, especially on AMF, + whose `header_insertion_mode` handling is the least-proven path. On any + breakage: note the GPU and driver version, collect logs, and leave the + toggle off. +- [ ] **E3 Fallback.** On a GPU without an HEVC encoder, the toggle warns + once in the log and keeps streaming H.264 with no error loop. ## F. Extended display & resolution match (8 min) -- [ ] **F1 — Lifecycle**: select "Extended display (iPad)" + Restart stream - with the iPad connected. *Expect*: a new display appears in Windows - Display settings, windows drag onto it, and the iPad shows it. Disconnect - the iPad. *Expect*: the virtual display disappears within ~5 s - (client-lost teardown). Quit/relaunch/crash never strands a phantom - monitor. -- [ ] **F2 — Native resolution**: with "Match extended display to the iPad's - resolution" on (default), the virtual display's mode equals the iPad's - native landscape resolution (e.g. 2420×1668) — the iPad picture is - edge-to-edge, no letterbox. Check +- [ ] **F1 Lifecycle.** Select "Extended display (iPad)" and Restart stream + with the iPad connected. Expect: a new display appears in Windows Display + settings, windows drag onto it, and the iPad shows it. Disconnect the + iPad. Expect: the virtual display disappears within about 5 s. Quit, + relaunch, or crash must never strand a phantom monitor. +- [ ] **F2 Native resolution.** With "Match extended display to the iPad's + resolution" on (the default), the virtual display's mode equals the + iPad's native landscape resolution (for example 2420×1668) and the iPad + picture is edge-to-edge with no letterbox. Check that `C:\VirtualDisplayDriver\vdd_settings.xml` exists and lists that mode - first. Toggle the match off + restart. *Expect*: driver default mode - (letterboxed picture is fine here). -- [ ] **F3 — 120 Hz mode** (ProMotion iPad): with the match on, Windows - offers the panel refresh (or falls back to the 60 Hz variant without - erroring). - -## G. Encoder deep checks (per vendor; ~5 min each) - -- [ ] **G1 — Idle VBV (real PTS)**: leave a static desktop for 60 s. - *Expect*: bandwidth on the Performance tab collapses (keepalive only), and - the first motion afterwards is clean, not a smear. If pacing looks wrong - on NVENC/AMF, retry with `set ETERNAL_LEGACY_PTS=1` and report — that - escape hatch existing is why this item is here. -- [ ] **G2 — AMF specifics** (AMD box): startup shows a keyframe (no black + first. Toggle the match off and restart. Expect: the driver's default + mode (a letterboxed picture is fine here). +- [ ] **F3 120 Hz mode** (ProMotion iPad). With the match on, Windows + offers the panel refresh, or falls back to the 60 Hz variant without + erroring. + +## G. Encoder deep checks (per vendor, about 5 min each) + +- [ ] **G1 Idle VBV (real PTS).** Leave a static desktop for 60 s. Expect: + bandwidth on the Performance tab collapses to keepalives, and the first + motion afterwards is clean, not a smear. If pacing looks wrong on NVENC + or AMF, retry with `set ETERNAL_LEGACY_PTS=1` and report; that escape + hatch existing is why this item is here. +- [ ] **G2 AMF specifics** (AMD box). Startup shows a keyframe (no black screen), recovery after loss works, and 10 minutes of streaming shows no periodic freeze. If broken: `set ETERNAL_AMF_DIAG=1`, reproduce, send `%APPDATA%\EternalMonitor\diagnostics\`. -- [ ] **G3 — High refresh**: `set ETERNAL_FPS=120` on a ProMotion iPad. - *Expect*: HUD ~100+ fps on a strong network, no capture-side stutter. -- [ ] **G4 — Stop/Start**: GUI Stop then Start. *Expect*: clean halt and a - fresh stream the iPad resumes automatically. +- [ ] **G3 High refresh.** `set ETERNAL_FPS=120` with a ProMotion iPad. + Expect: HUD at 100+ fps on a strong network with no capture-side stutter. +- [ ] **G4 Stop/Start.** GUI Stop, then Start. Expect: a clean halt and a + fresh stream that the iPad resumes automatically. ## H. Long soak (run in the background of the above) -- [ ] **H1**: keep one stream up 30+ minutes. *Expect*: no leak-shaped - memory growth on either end (Task Manager / Xcode gauge), no thermal - shutdown of the stream, HUD stats stay sane. +- [ ] **H1.** Keep one stream up 30+ minutes. Expect: no leak-shaped memory + growth on either end (Task Manager, Xcode gauge), no thermal shutdown of + the stream, and sane HUD stats throughout. ## When something fails -1. Host: **Copy logs** (Stream tab) or grab +1. Host: Copy logs (Stream tab) or grab `%APPDATA%\EternalMonitor\logs\eternal-host-session.log`. -2. iPad: Settings → the diagnostics list (most recent events), plus what the - screen showed. -3. Note GPU model + driver version, WiFi band, and which runbook item. -4. AMD encode issues: also `%APPDATA%\EternalMonitor\diagnostics\` with - `ETERNAL_AMF_DIAG=1` set. +2. iPad: the diagnostics list in Settings (most recent events), plus what + the screen showed. +3. Note the GPU model and driver version, the WiFi band, and which runbook + item failed. +4. AMD encode issues: also send `%APPDATA%\EternalMonitor\diagnostics\` + captured with `ETERNAL_AMF_DIAG=1`. Fixes land, the failing items get re-run, and only then does `v0.2.0` get -tagged (the tag builds and publishes the installer automatically). +tagged. The tag builds and publishes the installer automatically. diff --git a/README.md b/README.md index 93ffdf7..3c7cdea 100644 --- a/README.md +++ b/README.md @@ -1,66 +1,67 @@ # EternalMonitor -Use your iPad as a wireless second display for Windows — and control the PC from it. +Use your iPad as a wireless second display for Windows, and control the PC from it. [![CI](https://github.com/whoisaldo/EternalMonitor/actions/workflows/ci.yml/badge.svg)](https://github.com/whoisaldo/EternalMonitor/actions/workflows/ci.yml) [![Release](https://img.shields.io/github/v/release/whoisaldo/EternalMonitor?labelColor=111&color=e8ff47)](https://github.com/whoisaldo/EternalMonitor/releases/latest) [![Website](https://img.shields.io/badge/website-eternalmonitor.dev-e8ff47?style=flat&labelColor=111)](https://eternalmonitor.dev) -A Rust host on the PC captures the desktop with DXGI, encodes with the GPU +A Rust host on the PC captures the desktop with DXGI, encodes on the GPU (NVENC/AMF/QSV, H.264 or opt-in HEVC), and streams over UDP on the local network. A native Swift app on the iPad decodes with VideoToolbox and renders -with Metal. Touch, Apple Pencil, and two-finger scroll are relayed back as -mouse input. MIT licensed. +with Metal. Touch, Apple Pencil, and two-finger scroll travel back as mouse +input. MIT licensed. -**Contributions welcome** — transport, encoders, rendering, docs, anything. -Ping `aldobenches285` on Discord to collaborate. +Contributions welcome. Transport, encoders, rendering, docs, anything. Ping +`aldobenches285` on Discord to collaborate. ## What v0.2.0 does -- **Mirror or extend**: mirror the primary display, capture a specific - monitor, or stream a managed *virtual* extended display that exists only - while the iPad is connected — offered at the iPad's native resolution and - refresh rate. -- **Control the PC from the iPad**: tap to click, drag to move the mouse, two - fingers to scroll, hold for a right-click, Apple Pencil with pressure. - Negotiated per session; a view-only toggle is one switch away. -- **Protocol v2**: a real session (handshake with capability negotiation, busy - rejection, liveness), host heartbeats, client keyframe requests, receiver - reports, and NTP-style clock sync for an honest end-to-end latency readout. -- **Reliability**: adaptive bitrate (the host slider is the ceiling), packet +- **Mirror or extend.** Mirror the primary display, capture a specific + monitor, or stream a managed virtual extended display that exists only + while the iPad is connected. The virtual display comes up at the iPad's + native resolution and refresh rate. +- **Control the PC from the iPad.** Tap to click, drag to move the mouse, + two fingers to scroll, hold for a right-click, Apple Pencil with pressure. + Negotiated per session, with a view-only toggle in the app. +- **Protocol v2.** A real session (handshake with capability negotiation, + busy rejection, liveness), host heartbeats, client keyframe requests, + receiver reports, and NTP-style clock sync. The latency number in the HUD + is measured, not guessed. +- **Reliability.** Adaptive bitrate under the host slider's ceiling, packet pacing on keyframe bursts, keyframe recovery after loss, automatic reconnect after signal loss, and supervisor-driven crash recovery on the - host (an encoder crash restarts the pipeline in ~1 s without dropping the - session). -- **Codecs**: H.264 everywhere; HEVC/H.265 as an experimental opt-in + host. An encoder crash restarts the pipeline in about a second without + dropping the session. +- **Codecs.** H.264 everywhere. HEVC/H.265 is an experimental opt-in ("Prefer HEVC" in host Settings) with live mid-session switching. Not in scope yet: audio, USB transport, a first-party display driver (the extended display uses the bundled MIT-licensed [Virtual-Display-Driver](https://github.com/VirtualDrivers/Virtual-Display-Driver)), -and an App Store listing (the iPad app is TestFlight/Xcode-installed). +and an App Store listing. The iPad app installs via TestFlight or Xcode. ## Install (testers) Grab **EternalMonitor-Setup.exe** from the [latest release](https://github.com/whoisaldo/EternalMonitor/releases/latest), -run it, approve the one UAC prompt, and allow the firewall prompt (both -Private and Public). Step-by-step tester instructions live in +run it, approve the one UAC prompt, and allow the firewall prompt for both +Private and Public networks. Step-by-step tester instructions live in [scripts/QUICKSTART.txt](scripts/QUICKSTART.txt). The iPad app comes from TestFlight (ask for an invite) or an Xcode build. -SmartScreen note: the installer is not code-signed yet — "More info" → -"Run anyway". +SmartScreen note: the installer is not code-signed yet. Click "More info", +then "Run anyway". ## Build from source -The workspace is two Rust crates (`host/`, `proto/` = the pure `eternal-wire` -protocol crate) plus the Swift app in `ios/`. +The workspace is two Rust crates (`host/`, plus `proto/` which builds the +pure `eternal-wire` protocol crate) and the Swift app in `ios/`. ### Windows host Requirements: Rust stable (MSVC), an FFmpeg **7.1 shared** SDK, LLVM/libclang -(for bindgen). +for bindgen. ```powershell # Point the build at your FFmpeg 7.1 shared SDK (folder containing bin\avcodec-*.dll) @@ -70,12 +71,12 @@ cargo build --release -p eternal-host ``` `scripts\build-installer.ps1` builds the full Setup.exe (needs Inno Setup and -the same `FFMPEG_DIR`); `scripts\package.ps1` builds the bare zip. +the same `FFMPEG_DIR`). `scripts\package.ps1` builds the bare zip. ### macOS development loop (no Windows required) -The host builds and runs on macOS with a synthetic capture source — the whole -protocol, encoder, transport, and supervisor stack is exercised for real: +The host builds and runs on macOS with a synthetic capture source. The +protocol, encoder, transport, and supervisor stack all run for real: ```bash brew install ffmpeg@7 pkgconf xcodegen @@ -96,8 +97,8 @@ xcodebuild test -project EternalMonitor.xcodeproj -scheme EternalMonitor \ -destination 'platform=iOS Simulator,name=iPad Pro 11-inch (M4)' ``` -Open the generated project in Xcode to run on a physical iPad (your own -signing team). +Open the generated project in Xcode to run on a physical iPad with your own +signing team. ### Full-system test on one Mac @@ -107,23 +108,25 @@ EM_CODEC=hevc ./scripts/e2e_ios.sh # same, over HEVC ``` The harness launches the headless host and the simulator app, auto-connects, -and asserts ≥120 decoded frames at the right resolution via the app's -machine-readable log milestones. +and asserts at least 120 decoded frames at the right resolution via the +app's machine-readable log milestones. ## How it's tested -- **Golden wire vectors** (`proto/testdata/`) parsed byte-for-byte by both the +- Golden wire vectors (`proto/testdata/`) parsed byte-for-byte by both the Rust and Swift codecs, plus fuzz "never crashes" tests on both sides. -- **Pure-logic unit tests** with injected clocks: session machine, ABR ladder, +- Pure-logic unit tests with injected clocks: session machine, ABR ladder, pacer, reassembly, input mapping, gesture state machine, supervisor. -- **End-to-end tests** that run the real pipeline: Rust E2E (handshake, lossy - ABR step-down, encoder-crash recovery, input relay, HEVC negotiation) and - the simulator harness above. -- **CI** on every PR: Linux (wire crate), Windows (full host against pinned - FFmpeg 7.1), macOS (full workspace incl. E2E), and the iOS simulator suite. +- End-to-end tests that run the real pipeline. The Rust E2E covers + handshake, lossy ABR step-down, encoder-crash recovery, input relay, and + HEVC negotiation; the simulator harness above covers the full system. +- CI on every PR: Linux (wire crate), Windows (full host against pinned + FFmpeg 7.1), macOS (full workspace including E2E), and the iOS simulator + suite. -What CI can't verify — real GPU encoders, the virtual display driver, real -WiFi — is covered by a hardware runbook before each release. +CI cannot verify real GPU encoders, the virtual display driver, or real +WiFi. [HARDWARE_VERIFICATION.md](HARDWARE_VERIFICATION.md) is the runbook +that covers those before each release. ## Repo layout @@ -143,34 +146,35 @@ docs/ eternalmonitor.dev website (GitHub Pages) | `ETERNAL_ENCODER` | Force an encoder (`h264_nvenc`, `h264_amf`, `h264_qsv`, `libx264`) | | `ETERNAL_HEVC` | `1`/`0` overrides the HEVC preference (automation) | | `ETERNAL_FPS` | Override target FPS | -| `ETERNAL_CAPTURE` | `synthetic` = generated test pattern instead of DXGI | -| `ETERNAL_HEADLESS` | `1` = run without the GUI until SIGTERM/SIGINT | +| `ETERNAL_CAPTURE` | `synthetic` swaps DXGI for a generated test pattern | +| `ETERNAL_HEADLESS` | `1` runs without the GUI until SIGTERM/SIGINT | | `ETERNAL_VDD_TIMEOUT_SECS` | Virtual-display attach timeout | | `ETERNAL_ABR` | `0` disables adaptive bitrate | | `ETERNAL_DROP` | Test-only: inject fractional datagram loss | -| `ETERNAL_AMF_DIAG` | `1` = write AMF bitstream diagnostics | -| `ETERNAL_LEGACY_PTS` | `1` = old frame-counter PTS (escape hatch) | +| `ETERNAL_AMF_DIAG` | `1` writes AMF bitstream diagnostics | +| `ETERNAL_LEGACY_PTS` | `1` restores the old frame-counter PTS (escape hatch) | ## Troubleshooting -- **iPad can't connect**: same WiFi (not a guest network), firewall allowed - for Private *and* Public, manual IP entry beats discovery on tricky +- iPad can't connect: same WiFi (not a guest network), firewall allowed for + Private and Public, and manual IP entry beats discovery on tricky networks. The host window shows the address and a QR code. -- **Choppy video**: almost always WiFi. Get near the router, prefer 5 GHz, - wire the PC. The HUD's loss% and the host's ABR rung tell the story. -- **"H.264 (x264)" on the Stream tab**: the hardware encoder failed to open - and the host fell back to CPU encoding — update GPU drivers and restart the - stream. -- **Version mismatch**: protocol v2 is a clean break. A v0.1.x app or host +- Choppy video is almost always WiFi. Get near the router, prefer 5 GHz, + wire the PC. The HUD's loss% and the host's bitrate readout tell the + story. +- "H.264 (x264)" on the Stream tab means the hardware encoder failed to + open and the host fell back to CPU encoding. Update GPU drivers and + restart the stream. +- Version mismatch: protocol v2 is a clean break. A v0.1.x app or host shows a clear "update the other side" message instead of streaming. ## Reference docs -- [ARCHITECTURE.md](ARCHITECTURE.md) — the pipeline, protocol v2, and design -- [DECISIONS.md](DECISIONS.md) — why things are the way they are +- [ARCHITECTURE.md](ARCHITECTURE.md) covers the pipeline, protocol v2, and design +- [DECISIONS.md](DECISIONS.md) explains why things are the way they are - [RELEASE_NOTES.md](RELEASE_NOTES.md) -- [FRIENDS_TESTING.md](FRIENDS_TESTING.md) — organizer notes for beta testing -- [HARDWARE_VERIFICATION.md](HARDWARE_VERIFICATION.md) — the pre-release +- [FRIENDS_TESTING.md](FRIENDS_TESTING.md) has organizer notes for beta testing +- [HARDWARE_VERIFICATION.md](HARDWARE_VERIFICATION.md) is the pre-release runbook for everything CI can't prove ## Credits @@ -182,4 +186,4 @@ Built by Ali Younes ([@whoisaldo](https://github.com/whoisaldo)). ## License -Released under the MIT License — see [LICENSE](LICENSE). © 2026 Ali Younes. +Released under the MIT License. See [LICENSE](LICENSE). © 2026 Ali Younes. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 495aa58..2643a4a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,14 +1,14 @@ ## EternalMonitor v0.2.0 -A ground-up revamp of the streaming core. **Clean break: the v0.2.0 host and -iPad app only work with each other** — each side shows a clear "update the +A ground-up revamp of the streaming core. Clean break: the v0.2.0 host and +iPad app only work with each other. Each side shows a clear "update the other half" message if it meets a v0.1.x peer. ### Control the PC from the iPad - Tap to click, drag to move the mouse, two-finger scroll, half-second hold for a right-click, Apple Pencil with pressure. On by default ("Control PC - with touch" in the iPad Settings), negotiated per session — the host never - injects for a session that didn't ask. + with touch" in the iPad Settings) and negotiated per session; the host + never injects for a session that didn't ask. - While control is on, a three-finger tap toggles the stats HUD. ### Protocol v2 @@ -16,28 +16,28 @@ other half" message if it meets a v0.1.x peer. second device, instant reconnect takeover, liveness tracking, and clean goodbyes (including when the app is backgrounded). - Host heartbeats, client receiver reports, keyframe requests, and NTP-style - clock sync — the HUD's latency number is now a real end-to-end measurement. + clock sync. The HUD's latency number is now a real end-to-end measurement. - Media is raw Annex B in a fixed 32-byte header; FlatBuffers is gone. ### Reliability -- Adaptive bitrate: the host slider is now the **ceiling**; the stream steps +- Adaptive bitrate: the host slider is now the ceiling, and the stream steps down under loss and back up when the network recovers. - Keyframe recovery after loss (client-requested, host rate-limited), packet pacing on keyframe bursts, and automatic reconnect with backoff after "SIGNAL LOST". - Host supervisor v2: an encoder crash auto-restarts the pipeline in about a - second and the iPad resumes on the same session — no reconnect, no + second and the iPad resumes on the same session, with no reconnect and no re-handshake. Wedge watchdogs catch silent stalls. ### Video -- **HEVC/H.265** as an experimental opt-in ("Prefer HEVC" on the host): +- HEVC/H.265 as an experimental opt-in ("Prefer HEVC" on the host): negotiated per client, live mid-session codec switching, automatic H.264 fallback. - Real capture-time PTS (rate control finally sees true frame cadence), NV12 decode output with proper BT.601/709 handling, aspect-fit rendering, and draw-on-demand (no more free-running 120 Hz redraw). -- The extended (virtual) display now offers the **iPad's native resolution - and refresh rate**, and tears down when the client disconnects. +- The extended (virtual) display now offers the iPad's native resolution + and refresh rate, and tears down when the client disconnects. ### Quality of life - Settings apply from the first frame (including headless runs), atomic From b1c323daf0304c6986b622d8b6cfbda1616faa7f Mon Sep 17 00:00:00 2001 From: whoisaldo Date: Wed, 26 Aug 2026 13:15:02 -0400 Subject: [PATCH 3/3] Version 0.2.0 (host + wire crate) Single-sourced: the GUI banner and mDNS TXT read env!(CARGO_PKG_VERSION), the packaging scripts parse host/Cargo.toml, and iOS MARKETING_VERSION matches at 0.2.0 (build 5). Claude-Session: https://claude.ai/code/session_013ezpmTwAW6yAcRdy2DEaex --- Cargo.lock | 2 +- host/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90252e8..a4621bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1068,7 +1068,7 @@ checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "eternal-host" -version = "0.1.2" +version = "0.2.0" dependencies = [ "eframe", "eternal-wire", diff --git a/host/Cargo.toml b/host/Cargo.toml index f38a7b5..0f49e05 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "eternal-host" -version = "0.1.2" +version = "0.2.0" edition = "2021" [dependencies]