From 7b1d0f304d80e77e3e34e7ebf4ac6077ec938a40 Mon Sep 17 00:00:00 2001 From: Joey Stanford Date: Sun, 16 Aug 2026 17:48:50 -0600 Subject: [PATCH 1/7] feat(reticulum): add Ratspeak-parity LXMF voice memos Wire FIELD_AUDIO/AM_OPUS_OGG end-to-end (sidecar encode+send/receive, dedicated IPC, DM record/play UI) with Direct-only oversize-for-PN UX instead of PN-outage failures. --- docs/agents/reticulum.md | 3 +- docs/reticulum.md | 4 +- reticulum-sidecar/Cargo.lock | 16 + reticulum-sidecar/Cargo.toml | 1 + reticulum-sidecar/src/api/mod.rs | 14 + reticulum-sidecar/src/api/voice_memo.rs | 69 ++++ reticulum-sidecar/src/stack/live.rs | 215 ++++++++++- reticulum-sidecar/src/stack/lxmf_outbound.rs | 97 +++++ reticulum-sidecar/src/stack/mod.rs | 71 ++++ reticulum-sidecar/src/stack/persistence.rs | 1 + reticulum-sidecar/src/stack/types.rs | 11 + reticulum-sidecar/src/stack/voice_memo.rs | 345 ++++++++++++++++++ src/main/index.ts | 25 ++ src/main/ipc/reticulum-handlers.ts | 81 ++++ ...eticulum-proxy-rate-limit.contract.test.ts | 16 +- src/main/reticulum-attachment-audio.ts | 64 ++++ src/preload/index.ts | 15 + src/renderer/App.tsx | 46 +++ src/renderer/components/ChatComposer.tsx | 77 +++- src/renderer/components/ChatPanel.tsx | 27 +- .../ReticulumMessageStatusBadge.test.tsx | 13 +- .../ReticulumMessageStatusBadge.tsx | 9 +- .../ReticulumVoiceMemoLine.test.tsx | 55 +++ .../components/ReticulumVoiceMemoLine.tsx | 193 ++++++++++ src/renderer/lib/ingest/reticulumIngest.ts | 3 + src/renderer/lib/radio/BaseRadioProvider.ts | 5 + .../lib/radio/protocol-capabilities.test.ts | 4 + .../applyReticulumOutboundDeliveryStatus.ts | 8 +- .../lib/reticulum/computeWaveform.test.ts | 39 ++ src/renderer/lib/reticulum/computeWaveform.ts | 52 +++ .../reticulumAudioAttachmentCache.ts | 31 ++ .../lib/reticulum/reticulumIngestMerge.ts | 18 +- .../lib/reticulum/reticulumVoiceMemo.ts | 252 +++++++++++++ .../reticulumVoiceMemoPipeline.test.ts | 162 ++++++++ .../lib/reticulum/sendReticulumVoiceMemo.ts | 239 ++++++++++++ src/renderer/lib/storeRecordAdapters.ts | 19 + src/renderer/lib/types.ts | 6 + src/renderer/locales/cs/translation.json | 15 +- src/renderer/locales/de/translation.json | 15 +- src/renderer/locales/en/translation.json | 15 +- src/renderer/locales/es/translation.json | 15 +- src/renderer/locales/fr/translation.json | 15 +- src/renderer/locales/id/translation.json | 15 +- src/renderer/locales/it/translation.json | 15 +- src/renderer/locales/ja/translation.json | 15 +- src/renderer/locales/ko/translation.json | 15 +- src/renderer/locales/nl/translation.json | 15 +- src/renderer/locales/pl/translation.json | 15 +- src/renderer/locales/pt-BR/translation.json | 15 +- src/renderer/locales/ru/translation.json | 15 +- src/renderer/locales/tr/translation.json | 15 +- src/renderer/locales/uk/translation.json | 15 +- src/renderer/locales/zh/translation.json | 15 +- ...ticulumRuntime.reconnect-hardening.test.ts | 2 +- src/renderer/runtime/useReticulumRuntime.ts | 14 + src/renderer/stores/messageStore.ts | 9 + .../stores/reticulumVoiceMemoStore.test.ts | 67 ++++ .../stores/reticulumVoiceMemoStore.ts | 76 ++++ src/renderer/vitest.electronApiMock.ts | 13 + src/shared/electron-api.types.ts | 27 ++ src/shared/reticulum-voice-memo-types.test.ts | 34 ++ src/shared/reticulum-voice-memo-types.ts | 118 ++++++ 62 files changed, 2855 insertions(+), 51 deletions(-) create mode 100644 reticulum-sidecar/src/api/voice_memo.rs create mode 100644 reticulum-sidecar/src/stack/voice_memo.rs create mode 100644 src/main/reticulum-attachment-audio.ts create mode 100644 src/renderer/components/ReticulumVoiceMemoLine.test.tsx create mode 100644 src/renderer/components/ReticulumVoiceMemoLine.tsx create mode 100644 src/renderer/lib/reticulum/computeWaveform.test.ts create mode 100644 src/renderer/lib/reticulum/computeWaveform.ts create mode 100644 src/renderer/lib/reticulum/reticulumAudioAttachmentCache.ts create mode 100644 src/renderer/lib/reticulum/reticulumVoiceMemo.ts create mode 100644 src/renderer/lib/reticulum/reticulumVoiceMemoPipeline.test.ts create mode 100644 src/renderer/lib/reticulum/sendReticulumVoiceMemo.ts create mode 100644 src/renderer/stores/reticulumVoiceMemoStore.test.ts create mode 100644 src/renderer/stores/reticulumVoiceMemoStore.ts create mode 100644 src/shared/reticulum-voice-memo-types.test.ts create mode 100644 src/shared/reticulum-voice-memo-types.ts diff --git a/docs/agents/reticulum.md b/docs/agents/reticulum.md index 3b061c977..e5ca459e6 100644 --- a/docs/agents/reticulum.md +++ b/docs/agents/reticulum.md @@ -28,8 +28,9 @@ Deep subsystem reference for AI assistants. Open this when a task touches the Re - **Nomad errors:** `lib/nomad/nomadPageErrorHumanize.ts` (sidecar error codes → i18n); LinkClient Nomad overlay in `reticulum-sidecar/patches/` - **Nomad request bodies:** MessagePack `field_*`/`var_*` encode/decode lives in rsNomad `nomad-core` (`encode_request_fields` / `decode_request_fields`); sidecar `nomad_request_payload.rs` only unwraps HTTP base64 JSON before calling encode - **LXST voice:** `hasLxstVoice` gates Call buttons (Peers + Chat DM). Session helpers in `reticulumVoiceSession.ts` (dial/answer/hangup + mic PCM); UI store `reticulumVoiceStore.ts`; overlay `ReticulumVoiceOverlay` (App mount). Dedicated IPC `reticulum:voiceSendAudio` + push channel `reticulum:voiceAudio` (`/ws/voice`; preload `onVoiceAudio`); control via `electronAPI.reticulum.voice.*`. Runtime WS: `voice.update` / `voice.incoming` / `voice.stats` / `voice.terminated` / `voice.error` (errors should carry `link_id` when known; match by link/generation/remote). **Establish-only media:** Answer warms AudioContext; mic capture/TX starts only after `established`; sidecar soft-drops pre-establish PCM (`not_established`). Outbound progress tones: dial → peer DTMF fold → UK double-ring (`reticulumVoiceCallTones.ts` / `reticulumVoiceOutcome.ts` / `reticulumVoiceFeedback.ts`); media-start coalesces by `callGeneration` to avoid Answer mic thrash. Terminal reasons: treat sidecar `established`/`terminated` as completed (not fail). +- **LXMF voice memos:** `hasReticulumVoiceMemo` gates Chat DM mic (not LXST Call). Sidecar encodes Opus/`AM_OPUS_OGG` via `/api/v1/voice/memo/*` + dedicated IPC `electronAPI.reticulum.voiceMemo.*` (proxy blocked); Ogg capped ~240 KiB under the 256 KiB LXMF field / default PN deposit. Ingest stamps `FIELD_AUDIO` → attachment jail + `ReticulumVoiceMemoLine` playback (`chat:readReticulumAttachmentBytes`). Oversize-for-PN emits `message_too_large_for_propagation` (notice UX — never a PN-outage toast). - **LRGP games:** `hasLrgpGames` gates Games tab + Challenge (Peers / Chat DM). Sidecar `games_session` + `LrgpStore`; companion `games_outbound.db` persists last envelope + `delivery_state` (LXMF outbound bridge → session chips / Resend). Dedicated IPC `electronAPI.reticulum.games.*` / `reticulum:games*` (proxy rejects `/api/v1/games/*`); WS `games.update` / `games.action_result`. Parity: [../reticulum-games-parity.md](../reticulum-games-parity.md). -- **Gating:** `hasReticulumDiscoveryMap` (Map tab); `hasReticulumRemotePanel` / `hasRncpTransfer` (Remote tab + Chat DM rncp); `hasRrcPanel` (RRC tab); `hasLxstVoice` (LXST Call); `hasLrgpGames` (Games); `hasReticulumInterfaceConfig` / `hasReticulumNetworkPanel` / `ProtocolCapabilities` +- **Gating:** `hasReticulumDiscoveryMap` (Map tab); `hasReticulumRemotePanel` / `hasRncpTransfer` (Remote tab + Chat DM rncp); `hasRrcPanel` (RRC tab); `hasLxstVoice` (LXST Call); `hasReticulumVoiceMemo` (LXMF voice memos); `hasLrgpGames` (Games); `hasReticulumInterfaceConfig` / `hasReticulumNetworkPanel` / `ProtocolCapabilities` - **rnsh/rncp:** sidecar `stack/{rnsh_session,rncp_transfer,path_speed,link_task}.rs` + HTTP `/api/v1/rnsh/*`, `/api/v1/rncp/*`, `/api/v1/remote/*`; typed `electronAPI.reticulum.rnsh|rncp|remote`; picker-gated send/fetch paths in `reticulum-remote-paths.ts`; LXMF enable-request sentinel `mesh-client:request-rncp-receive:v1` (`rncpRequestEnable.ts`); peer reply `mesh-client:rncp-receive-dest:v1:` autofills via `applyRncpReceiveDestShare` (prefer pending from `markRncpReceiveDestSharePending` / `sendRncpRequestEnable`; still apply without pending for older peers); enable-request modal + dest-share side effects deduped by LXMF `message_hash` (`rncpLxmfControlSideEffectDedup`) so catch-up cannot re-fire; already-listening auto-share is once per peer per request-enable cooldown; inbound listener config persists (`rncp_listener_*` in `mesh_client_stack.json`) and restores on live stack start - **Runtime:** `useReticulumRuntime`, `lib/sessions/reticulumSession.ts`, `lib/ingest/reticulumIngest.ts`; connect starts sidecar, not `ConnectionDriver` RF — marks **configured** when HTTP + identity ready (live attach may still run); `RETICULUM_CONFIGURED_EVENT` wakes RRC. Cancel/stop is fire-and-forget vs cargo/BLE (`START_ABORTED` checkpoints; next start does not rejoin a doomed promise). LXMF/RRC proxy sends: **15 s** `RETICULUM_IPC_SEND_TIMEOUT_MS`. RRC auto-connect (`useRrcStartupAutoConnect`): ~**500 ms** while hubs pending, ~4 s steady. Sidecar RRC: `rrc_codec` / `rrc_link` / `rrc_session` / `api/rrc.rs` - **Diagnostics:** `ReticulumDiagnosticEngine.ts` (Reticulum-native rows; no LoRa hop-goblin semantics) — includes `reticulum/sidecar-unhealthy` (60s grace; HTTP health, not listen-first ready lag), `reticulum/rns-not-ready` / `reticulum/lxmf-not-ready`, `reticulum/propagation-sync-stuck`, `reticulum/propagation-sync-failing` (1h TTL) diff --git a/docs/reticulum.md b/docs/reticulum.md index ae0137107..25c09a999 100644 --- a/docs/reticulum.md +++ b/docs/reticulum.md @@ -29,7 +29,7 @@ After changing interfaces on a live network, **restart the stack** so RNS picks | Stack lifecycle | Start / stop / auto-start; disconnect & quit. Sidecar **listen-first**: HTTP binds before live RNS/LXMF attach; Connect marks **configured** when HTTP + identity are ready (live attach / BLE may still be in progress) | | Interfaces | TCP client, I2P (`peers`), Auto discovery, RNode (USB serial, `ble://…`, Wi‑Fi `tcp://host:7633`); default hub picker by region (Primary & Global selected by default; added disabled; syncs/repairs selected endpoints and disables remaining decommissioned testnet hubs) | | Identity | Generate / import mnemonic; Ratspeak `.rsi` PIN backup; official raw identity file export/import; **identity vault** optional local `.rsi` store | -| LXMF chat | DM-only text and reactions (outbound LXMF file/voice-note attach removed; attachment labels render; **cached raster images** display inline; use Remote rncp for peer files). **LXST live voice Call** is separate telephony (rsLXST), not an LXMF voice clip. | +| LXMF chat | DM-only text, reactions, and **voice memos** (`FIELD_AUDIO` / `AM_OPUS_OGG`, Ratspeak wire parity; ~240 KiB / ~4 min cap under the 256 KiB LXMF field + default PN deposit). Attachment labels render; **cached raster images** display inline; use Remote rncp for peer files. **LXST live voice Call** is separate telephony (rsLXST), not an LXMF voice clip. Oversized memos that exceed the PN deposit limit surface as Direct-only (`message_too_large_for_propagation`) — never as a PN outage. | | Remote | **rnsh** multi-session shell + **rncp** send/receive/fetch under one tab (Shell / Transfer / Saved / Settings); Chat DM send-file convenience; path-speed gate (TCP/network); inbound Ask/allow-list; auto-reconnect / auto-retry; LXMF “request enable receive” prompt between mesh-client peers | | RRC | Reticulum Relay Chat — discovered/manual/favourite hubs, up to **8** concurrent sessions, hub/room auto-join, rooms, nicklists, slash commands (`/list`, `/who`, `/join`, …), unread badges + sounds for any new room message by default (also badges the **Reticulum protocol pill** with LXMF Chat; **App → Notifications** can switch to IRC-style DMs/@mentions only), toasts when the RRC tab is inactive, automatic reconnect with backoff | | Delivery | **Direct** when destination is in path table (outbound-initiated Direct replies need the sidecar **outbound Direct backchannel**). After Direct exhausts: **multi-PN cascade** — preferred remote → other enabled remotes (hop-sorted) → in **Auto** only, up to 3 heard-but-not-added **Discovered** PNs (hop-sorted, never persisted) → **local-prop last**. Remote PN Completes as `delivered` (**Stored at propagation node**, PN + green check); local-prop Completes as `stored_locally` (deposited on your hosted PN — PN + amber house; full PN parity with `lxmd`, including peer `/offer` sync — not an outbox). Propagation mode **Off** disables the cascade entirely. **Paper** for offline encrypted QR/`lxm://` handoff (no network — Completes immediately, no `lxmf_outbound_status`). Path/transport badges (RF/BLE/TCP/NET, multi, PN, Paper) are egress evidence — network UI stays **Sending** until `lxmf_outbound_status` (`delivered` / `stored_locally` / `failed`). Terminal `delivery_status` + `delivery_method` persist in SQLite. Hosted-PN Completes ≠ recipient opened Chat. Inbound `received_via` / TCP badges use local interface **config type**, not display name. | @@ -348,7 +348,7 @@ When multiple enabled local RNode interfaces are connected, the interface list s ## Chat (LXMF) - **DM-only** on the Chat tab — no RF channel pills (RRC covers hub rooms separately) -- Text and emoji reactions. **Outbound LXMF file/voice attach is not offered** (removed); historic `[file:name:mime]` bubbles and inbound Sideband-style attachments render a read-only label; when the file remains in `reticulum/attachments/`, **raster images** (JPEG/PNG/GIF/WebP/AVIF/BMP — not SVG) display inline via main-process `chat:readReticulumAttachmentAsDataUrl` (magic-byte MIME check, 2 MiB cap, path jailed, IPC rate-limited). Peer file transfer is via Remote rncp. +- Text and emoji reactions. **LXMF voice memos** use native `FIELD_AUDIO` (Ogg/Opus `AM_OPUS_OGG`); Chat DM mic records via sidecar `/api/v1/voice/memo/*` (dedicated IPC). Historic `[file:name:mime]` bubbles and inbound Sideband-style attachments render a read-only label; when the file remains in `reticulum/attachments/`, **raster images** (JPEG/PNG/GIF/WebP/AVIF/BMP — not SVG) display inline via main-process `chat:readReticulumAttachmentAsDataUrl` (magic-byte MIME check, 2 MiB cap, path jailed, IPC rate-limited), and **voice memos** play via `chat:readReticulumAttachmentBytes` (OggS sniff, 256 KiB). Peer file transfer is via Remote rncp. - **Replies:** outbound DMs stamp LXMF `FIELD_REPLY_TO` (0x30) and optional `FIELD_REPLY_QUOTE` (0x31, capped) before sign so peers see structured replies; ingest/Chat use `reticulum_reply_to_hash` plus quote preview (store parent when present, else wire quote) and jump-to-parent by message hash - **Reactions:** outbound tapbacks stamp standard LXMF `FIELD_REACTION` (0x40) before sign — a msgpack map (`REACTION_TO` = 32-byte parent hash, `REACTION_CONTENT` = emoji) — while still carrying the emoji as message content, so Ratspeak/Sideband render a structured reaction and clients that ignore `0x40` still show the emoji. Inbound decode is fail-open: a valid `0x40` maps to `reaction_target` (accepting a 32-byte binary or 64-hex `REACTION_TO`, normalized to lowercase) and wins over a reply for classification; a missing or malformed field leaves the reply/plain-text path unchanged. Ratspeak legacy `0xFB`/`0xFC` reaction envelopes are not emitted or decoded. - Outbound **Sending** until sidecar emits `lxmf_outbound_status` (`delivered` / `stored_locally` / `failed`); `/api/v1/lxmf/send` may return `delivery_status: "queued"` or `"sending"` — that is enqueue/acceptance, not delivery confirmation. After Direct exhausts, the sidecar **cascades** preferred remote → other enabled remotes (hop-sorted) → in **Auto** only, up to 3 heard-but-not-added **Discovered** PNs (hop-sorted) → local-prop last, re-emitting `sending` with `delivery_method: "propagated"` (remote) or `"stored_locally"` (hosted local PN) between attempts. **Exception — paper:** Chat DM **Share as paper** (`createReticulumPaperMessage` → `POST /api/v1/lxmf/paper/create`) encrypts offline to a QR/`lxm://` URI with **no network send**; Completes immediately (`delivery_method: paper`, badge **Paper**) and does **not** use `lxmf_outbound_status`. Ingest via Chat **Scan paper**, Network **Scan / import**, or OS `lxm://` (`POST /api/v1/lxmf/paper/ingest` — HTTP `message` fallback-ingested when WS lags). Create needs peer pubkey (`identity_unknown` otherwise); ingest needs matching local identity (`decrypt_failed` otherwise); size-capped (`paper_too_large`). diff --git a/reticulum-sidecar/Cargo.lock b/reticulum-sidecar/Cargo.lock index 0eaf28ac9..f5af60467 100644 --- a/reticulum-sidecar/Cargo.lock +++ b/reticulum-sidecar/Cargo.lock @@ -352,6 +352,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.0" @@ -1572,6 +1578,7 @@ dependencies = [ "lxst-telephony", "nomad-core", "notify", + "ogg", "rand 0.8.6", "rmpv", "rns-crypto", @@ -1796,6 +1803,15 @@ dependencies = [ "objc2", ] +[[package]] +name = "ogg" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdab8dcd8d4052eaacaf8fb07a3ccd9a6e26efadb42878a413c68fc4af1dee2b" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/reticulum-sidecar/Cargo.toml b/reticulum-sidecar/Cargo.toml index 6a41dfa49..411e7537e 100644 --- a/reticulum-sidecar/Cargo.toml +++ b/reticulum-sidecar/Cargo.toml @@ -59,6 +59,7 @@ rmpv = { version = "1", optional = true } base64 = "0.22" bytes = "1" ciborium = "0.2" +ogg = "0.9" rand = "0.8" thiserror = "2" notify = { version = "6.1", optional = true, default-features = true } diff --git a/reticulum-sidecar/src/api/mod.rs b/reticulum-sidecar/src/api/mod.rs index dcd703239..2e23625ac 100644 --- a/reticulum-sidecar/src/api/mod.rs +++ b/reticulum-sidecar/src/api/mod.rs @@ -17,6 +17,7 @@ mod status; mod system; pub(crate) mod validate; mod voice; +mod voice_memo; mod ws; use std::sync::Arc; @@ -275,6 +276,19 @@ pub fn router(stack: Arc) -> Router { .route("/api/v1/voice/hangup", post(voice::voice_hangup)) .route("/api/v1/voice/mute", post(voice::voice_mute)) .route("/api/v1/voice/audio", post(voice::voice_audio)) + .route( + "/api/v1/voice/memo/start", + post(voice_memo::voice_memo_start), + ) + .route( + "/api/v1/voice/memo/audio", + post(voice_memo::voice_memo_audio), + ) + .route("/api/v1/voice/memo/stop", post(voice_memo::voice_memo_stop)) + .route( + "/api/v1/voice/memo/cancel", + post(voice_memo::voice_memo_cancel), + ) .route("/api/v1/games/status", get(games::games_status)) .route("/api/v1/games/apps", get(games::games_apps)) .route("/api/v1/games/sessions", get(games::games_sessions)) diff --git a/reticulum-sidecar/src/api/voice_memo.rs b/reticulum-sidecar/src/api/voice_memo.rs new file mode 100644 index 000000000..87e400d37 --- /dev/null +++ b/reticulum-sidecar/src/api/voice_memo.rs @@ -0,0 +1,69 @@ +//! Voice memo encode HTTP endpoints (dedicated IPC; not generic proxy). + +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use serde::Deserialize; + +use crate::api::validate::{MAX_DEST_HASH_CHARS, reject_oversize}; +use crate::stack::StackHandle; + +const MAX_SAMPLES_B64_CHARS: usize = 512 * 1024; +const MAX_SESSION_ID_CHARS: usize = 64; + +#[derive(Debug, Deserialize)] +pub struct VoiceMemoSessionBody { + pub session_id: String, +} + +#[derive(Debug, Deserialize)] +pub struct VoiceMemoAudioBody { + pub session_id: String, + #[serde(default = "default_channels")] + pub channels: u8, + pub samples_b64: String, +} + +fn default_channels() -> u8 { + 1 +} + +pub async fn voice_memo_start(State(stack): State>) -> Json { + Json(stack.voice_memo_start()) +} + +pub async fn voice_memo_audio( + State(stack): State>, + Json(body): Json, +) -> Json { + if let Some(err) = reject_oversize("session_id", &body.session_id, MAX_SESSION_ID_CHARS) { + return Json(serde_json::json!({ "ok": false, "error": err })); + } + if let Some(err) = reject_oversize("samples_b64", &body.samples_b64, MAX_SAMPLES_B64_CHARS) { + return Json(serde_json::json!({ "ok": false, "error": err })); + } + Json(stack.voice_memo_audio(&body.session_id, body.channels, &body.samples_b64)) +} + +pub async fn voice_memo_stop( + State(stack): State>, + Json(body): Json, +) -> Json { + if let Some(err) = reject_oversize("session_id", &body.session_id, MAX_SESSION_ID_CHARS) { + return Json(serde_json::json!({ "ok": false, "error": err })); + } + // Reuse dest-hash char budget as a generic short-id cap when session_id is huge. + let _ = MAX_DEST_HASH_CHARS; + Json(stack.voice_memo_stop(&body.session_id)) +} + +pub async fn voice_memo_cancel( + State(stack): State>, + Json(body): Json, +) -> Json { + if let Some(err) = reject_oversize("session_id", &body.session_id, MAX_SESSION_ID_CHARS) { + return Json(serde_json::json!({ "ok": false, "error": err })); + } + Json(stack.voice_memo_cancel(&body.session_id)) +} diff --git a/reticulum-sidecar/src/stack/live.rs b/reticulum-sidecar/src/stack/live.rs index 5283b9ac4..4b2d30d60 100644 --- a/reticulum-sidecar/src/stack/live.rs +++ b/reticulum-sidecar/src/stack/live.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use lxmf_core::constants::{ - DeliveryMethod, FIELD_FILE_ATTACHMENTS, FIELD_ICON_APPEARANCE, FIELD_REACTION, + AM_OPUS_OGG, DeliveryMethod, FIELD_FILE_ATTACHMENTS, FIELD_ICON_APPEARANCE, FIELD_REACTION, REACTION_CONTENT, REACTION_TO, }; use lxmf_core::message::LxMessage; @@ -24,6 +24,8 @@ const FIELD_REPLY_TO: u8 = 0x30; const FIELD_REPLY_QUOTE: u8 = 0x31; /// Cap wire quote length (matches renderer `REPLY_PREVIEW_MAX_LEN` without ellipsis). const REPLY_QUOTE_MAX_CHARS: usize = 50; +/// lxmf-core per-field unpack limit — audio above this cannot be received by peers. +const MAX_LXMF_AUDIO_FIELD_BYTES: usize = 256 * 1024; use lxmf_core::peer::OutboundOfferPolicy; use lxmf_core::router::LxmRouter; use rns_identity::destination::Destination; @@ -3227,6 +3229,30 @@ impl LiveBridge { method: DeliveryMethod, reply_to: Option<[u8; 32]>, reply_quote: Option<&str>, + ) -> Result<(LxMessage, String), String> { + self.prepare_signed_outbound_lxmf_with_audio( + dest, + title, + content, + method, + reply_to, + reply_quote, + None, + ) + } + + /// Like [`prepare_signed_outbound_lxmf`], optionally stamping native + /// `FIELD_AUDIO` (voice memo) before `sign()` so it is covered by the hash. + #[allow(clippy::too_many_arguments)] // mirrors prepare_signed_outbound_lxmf + optional audio bytes + fn prepare_signed_outbound_lxmf_with_audio( + &self, + dest: [u8; 16], + title: &str, + content: &str, + method: DeliveryMethod, + reply_to: Option<[u8; 32]>, + reply_quote: Option<&str>, + audio: Option<&[u8]>, ) -> Result<(LxMessage, String), String> { let mut msg = LxMessage::new( dest, @@ -3236,6 +3262,17 @@ impl LiveBridge { method, ); apply_reply_fields(&mut msg, reply_to, reply_quote); + if let Some(bytes) = audio { + if bytes.len() > MAX_LXMF_AUDIO_FIELD_BYTES { + return Err(format!( + "audio_too_large: {} > {}", + bytes.len(), + MAX_LXMF_AUDIO_FIELD_BYTES + )); + } + msg.set_audio_field(AM_OPUS_OGG, bytes) + .map_err(|e| format!("lxmf set_audio_field: {e:?}"))?; + } let signing_key = self .identity .get_signing_key() @@ -3612,6 +3649,10 @@ impl LiveBridge { if let Ok(mut node) = self.propagation.local_node().lock() { apply_pn_hosting_policy_to_node(&mut node, policy); } + if let Ok(mut driver) = self.outbound.lock() { + driver + .set_propagation_max_message_size(policy.propagation_limit_kb.saturating_mul(1024)); + } // Restart announce loop with updated interval / name when serving. if self.propagation.is_local_serving() { self.prop_announce.start( @@ -4457,13 +4498,20 @@ impl LiveBridge { .as_deref() .map(str::trim) .filter(|q| !q.is_empty()); - let (msg, message_hash_hex) = self.prepare_signed_outbound_lxmf( + let audio_bytes = decode_lxmf_audio_request(req.audio.as_ref())?; + let content = if req.text.trim().is_empty() && audio_bytes.is_some() { + "[voice:0]".to_string() + } else { + req.text.clone() + }; + let (msg, message_hash_hex) = self.prepare_signed_outbound_lxmf_with_audio( dest, "", - &req.text, + &content, delivery_method, reply_to, reply_quote, + audio_bytes.as_deref(), )?; let mut router = self.router.lock().await; router @@ -4481,7 +4529,7 @@ impl LiveBridge { let mut payload = serde_json::json!({ "sender_hash": self.lxmf_hash_hex, "sender_name": self.display_name, - "text": req.text, + "text": content, "timestamp": ts_ms, "to_hash": req.destination_hash, "reply_to_hash": reply_to_hash_echo, @@ -4501,6 +4549,11 @@ impl LiveBridge { ); } } + if let Some(ref bytes) = audio_bytes { + if let Some(obj) = payload.as_object_mut() { + obj.insert("audio".into(), audio_json_from_bytes(AM_OPUS_OGG, bytes)); + } + } if let Ok(mut driver) = self.outbound.lock() { driver.process_tick(&mut router, &self.event_tx); @@ -4518,7 +4571,7 @@ impl LiveBridge { Ok(serde_json::json!({ "ok": true, "destination_hash": req.destination_hash, - "text": req.text, + "text": content, "delivery_method": delivery_method_str, "sent_via": egress_via, "delivery_status": "queued", @@ -4923,6 +4976,18 @@ pub(super) fn lxmf_payload_from_message( obj.insert("attachment".into(), attachment); } } + if let Some(audio) = audio_json_from_message(msg) { + if let Some(obj) = payload.as_object_mut() { + let text_empty = obj + .get("text") + .and_then(|t| t.as_str()) + .is_none_or(|t| t.trim().is_empty()); + if text_empty { + obj.insert("text".into(), serde_json::Value::String("[voice:0]".into())); + } + obj.insert("audio".into(), audio); + } + } if let Some(icon) = icon_appearance_json_from_message(msg) { if let Some(obj) = payload.as_object_mut() { obj.insert("icon_appearance".into(), icon); @@ -5177,6 +5242,51 @@ fn attachment_json_from_message(msg: &LxMessage) -> Option { })) } +fn audio_json_from_bytes(mode: u8, bytes: &[u8]) -> serde_json::Value { + use base64::Engine as _; + serde_json::json!({ + "mode": mode, + "size_bytes": bytes.len(), + "data_base64": base64::engine::general_purpose::STANDARD.encode(bytes), + }) +} + +/// Decode native `FIELD_AUDIO` for WS/HTTP payloads. Malformed audio fails open +/// (returns `None`) without poisoning the rest of the message. +fn audio_json_from_message(msg: &LxMessage) -> Option { + let audio = msg.audio_field().ok()??; + Some(audio_json_from_bytes(audio.mode, audio.bytes)) +} + +fn decode_lxmf_audio_request( + audio: Option<&super::types::LxmfAudioRequest>, +) -> Result>, String> { + use base64::Engine as _; + let Some(audio) = audio else { + return Ok(None); + }; + if audio.mode != AM_OPUS_OGG { + return Err(format!( + "unsupported_audio_mode: {} (expected {AM_OPUS_OGG})", + audio.mode + )); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(audio.data_base64.as_bytes()) + .map_err(|e| format!("audio_base64_decode: {e}"))?; + if bytes.len() > MAX_LXMF_AUDIO_FIELD_BYTES { + return Err(format!( + "audio_too_large: {} > {}", + bytes.len(), + MAX_LXMF_AUDIO_FIELD_BYTES + )); + } + if bytes.is_empty() { + return Err("audio_empty".into()); + } + Ok(Some(bytes)) +} + #[allow(clippy::needless_pass_by_value)] // payload is moved into the broadcast frame pub(super) fn emit_lxmf_event(event_tx: &broadcast::Sender, payload: serde_json::Value) { let frame = serde_json::json!({ @@ -6816,6 +6926,101 @@ mod icon_appearance_tests { } } +#[cfg(test)] +mod audio_field_tests { + use super::*; + use base64::Engine as _; + use lxmf_core::constants::AM_OPUS_OGG; + use lxmf_core::message::LxMessage; + + #[test] + fn audio_json_from_message_round_trips_opus_ogg() { + let ogg = b"OggS\0fake-opus-bytes"; + let mut msg = LxMessage::new([0u8; 16], [1u8; 16], "", "", DeliveryMethod::Direct); + msg.set_audio_field(AM_OPUS_OGG, ogg).expect("set audio"); + let json = audio_json_from_message(&msg).expect("audio json"); + assert_eq!(json["mode"], AM_OPUS_OGG); + assert_eq!(json["size_bytes"], ogg.len()); + let decoded = base64::engine::general_purpose::STANDARD + .decode(json["data_base64"].as_str().expect("b64")) + .expect("decode"); + assert_eq!(decoded, ogg); + } + + #[test] + fn audio_json_from_message_malformed_fail_open() { + let mut msg = LxMessage::new([0u8; 16], [1u8; 16], "", "hi", DeliveryMethod::Direct); + // Intentionally malformed: array of 3 elements instead of [mode, bytes]. + msg.set_msgpack_field( + lxmf_core::constants::FIELD_AUDIO, + vec![0x93, AM_OPUS_OGG, 0xc4, 0x00, 0xc0], + ) + .expect("set malformed"); + assert!(audio_json_from_message(&msg).is_none()); + let payload = lxmf_payload_from_message( + &msg, + "aa".repeat(16).as_str(), + "Self", + None, + None, + "inbound", + None, + ); + assert_eq!(payload["text"], "hi"); + assert!(payload.get("audio").is_none()); + } + + #[test] + fn lxmf_payload_sets_voice_marker_when_text_empty() { + let ogg = b"OggS\0memo"; + let mut msg = LxMessage::new([0u8; 16], [1u8; 16], "", "", DeliveryMethod::Direct); + msg.set_audio_field(AM_OPUS_OGG, ogg).expect("set audio"); + let payload = lxmf_payload_from_message( + &msg, + "aa".repeat(16).as_str(), + "Self", + None, + None, + "inbound", + None, + ); + assert_eq!(payload["text"], "[voice:0]"); + assert!(payload.get("audio").is_some()); + assert!(payload.get("attachment").is_none()); + } + + #[test] + fn decode_lxmf_audio_request_rejects_oversize() { + let oversized = vec![0u8; MAX_LXMF_AUDIO_FIELD_BYTES + 1]; + let req = super::super::types::LxmfAudioRequest { + mode: AM_OPUS_OGG, + data_base64: base64::engine::general_purpose::STANDARD.encode(&oversized), + }; + let err = decode_lxmf_audio_request(Some(&req)).expect_err("oversize"); + assert!(err.starts_with("audio_too_large"), "{err}"); + } + + #[test] + fn set_audio_field_changes_message_hash_after_sign() { + use rns_identity::identity::Identity; + let identity = Identity::new(); + let signing = identity.get_signing_key().expect("signing key"); + let source = identity.hash; + + let mut without = + LxMessage::new([0u8; 16], source, "", "[voice:0]", DeliveryMethod::Direct); + without.sign(&signing).expect("sign"); + let hash_without = without.hash.expect("hash"); + + let mut with = LxMessage::new([0u8; 16], source, "", "[voice:0]", DeliveryMethod::Direct); + with.set_audio_field(AM_OPUS_OGG, b"OggS\0x") + .expect("set audio"); + with.sign(&signing).expect("sign"); + let hash_with = with.hash.expect("hash"); + assert_ne!(hash_without, hash_with); + } +} + #[cfg(test)] mod reply_field_tests { use super::*; diff --git a/reticulum-sidecar/src/stack/lxmf_outbound.rs b/reticulum-sidecar/src/stack/lxmf_outbound.rs index cbfad6b98..fb267b4c9 100644 --- a/reticulum-sidecar/src/stack/lxmf_outbound.rs +++ b/reticulum-sidecar/src/stack/lxmf_outbound.rs @@ -193,6 +193,8 @@ pub struct LxmfOutboundDriver { pending_pn_targets: HashMap<[u8; 32], [u8; 16]>, /// When local-prop is serving, cascade deposits go in-process (no self-Link). local_prop_node: Option>>, + /// Effective PN deposit size limit (from `propagation_limit_kb`). + propagation_max_message_size: usize, /// Local LXMF identity (retained for driver construction / future failed-detail payloads). #[allow(dead_code)] self_lxmf_hash: String, @@ -234,6 +236,8 @@ impl LxmfOutboundDriver { pending_pn_deposits: HashMap::new(), pending_pn_targets: HashMap::new(), local_prop_node: None, + propagation_max_message_size: + crate::stack::pn_hosting_policy::DEFAULT_PROPAGATION_LIMIT_KB.saturating_mul(1024), self_lxmf_hash: self_lxmf_hash.clone(), self_display_name, }; @@ -306,6 +310,11 @@ impl LxmfOutboundDriver { self.local_prop_node = node; } + /// Update the PN deposit size ceiling used for oversize preflight. + pub fn set_propagation_max_message_size(&mut self, max_bytes: usize) { + self.propagation_max_message_size = max_bytes.max(1); + } + pub fn known_identities_for_propagation(&self) -> HashMap { let mut out = self.known_identities.clone(); for (k, v) in &self.pinned_identities { @@ -701,6 +710,27 @@ impl LxmfOutboundDriver { } } }; + // Preflight vs PN max_message_size — rsLXMF rejects oversized deposits silently; + // surface a distinct terminal error so the UI never treats this as a PN outage. + let limit = self.propagation_max_message_size; + if packed.len() > limit { + tracing::warn!( + target: "lxmf-outbound", + prop = %prop_hex, + dest = %hex::encode(message.destination_hash), + size_bytes = packed.len(), + limit_bytes = limit, + "DeliverPropagated: message too large for propagation — terminal (no cascade)" + ); + self.emit_outbound_failed_too_large_for_propagation( + router, + event_tx, + message, + limit, + packed.len(), + ); + return; + } let hops = route_hops_for(&self.route_hops, prop_hash); let message_hash_hex = message.hash.as_ref().map(hex::encode); let transient_id_hex = message.transient_id.as_ref().map(hex::encode); @@ -1161,6 +1191,58 @@ impl LxmfOutboundDriver { None, None, Some(attempts), + None, + None, + None, + ); + } + } + + /// Terminal failure when packed size exceeds the PN deposit limit. + /// Does **not** advance the cascade — Direct was already tried; offline store is impossible. + fn emit_outbound_failed_too_large_for_propagation( + &mut self, + router: &mut LxmRouter, + event_tx: &broadcast::Sender, + mut message: LxMessage, + limit_bytes: usize, + size_bytes: usize, + ) { + message.mark_failed(); + let method = message + .hash + .or(message.message_id) + .map(|h| self.cascade_wire_delivery_method(h)) + .unwrap_or("propagated"); + tracing::warn!( + target: "lxmf-outbound", + dest = %hex::encode(message.destination_hash), + method, + limit_bytes, + size_bytes, + "LXMF outbound failed: message_too_large_for_propagation" + ); + if let Some(hash) = message.hash.or(message.message_id) { + let attempts = message.delivery_attempts; + self.clear_pn_cascade_state(hash); + self.direct_path_failovers.remove(&hash); + self.pending_pn_deposits.remove(&hash); + let _ = router.mark_outbound_failed(&hash); + emit_outbound_status_detailed_with_attempts( + event_tx, + Some(serde_json::Value::String(hex::encode(hash))), + Some(serde_json::Value::String(hex::encode( + message.destination_hash, + ))), + "failed", + Some(method), + None, + None, + None, + Some(attempts), + Some("message_too_large_for_propagation"), + Some(limit_bytes), + Some(size_bytes), ); } } @@ -1762,6 +1844,9 @@ fn emit_outbound_status_detailed( tried_interfaces, failover_rounds, None, + None, + None, + None, ); } @@ -1776,6 +1861,9 @@ fn emit_outbound_status_detailed_with_attempts( tried_interfaces: Option>, failover_rounds: Option, delivery_attempts: Option, + error: Option<&str>, + limit_bytes: Option, + size_bytes: Option, ) { let mut payload = serde_json::Map::new(); if let Some(h) = message_hash { @@ -1803,6 +1891,15 @@ fn emit_outbound_status_detailed_with_attempts( if let Some(attempts) = delivery_attempts { payload.insert("delivery_attempts".into(), serde_json::json!(attempts)); } + if let Some(err) = error { + payload.insert("error".into(), serde_json::Value::String(err.into())); + } + if let Some(limit) = limit_bytes { + payload.insert("limit_bytes".into(), serde_json::json!(limit)); + } + if let Some(size) = size_bytes { + payload.insert("size_bytes".into(), serde_json::json!(size)); + } let frame = serde_json::json!({ "type": "lxmf_outbound_status", "payload": payload, diff --git a/reticulum-sidecar/src/stack/mod.rs b/reticulum-sidecar/src/stack/mod.rs index fe5841cc3..0f1f34f74 100644 --- a/reticulum-sidecar/src/stack/mod.rs +++ b/reticulum-sidecar/src/stack/mod.rs @@ -71,6 +71,8 @@ mod rrc_link; #[cfg(feature = "rns-stack")] mod rrc_session; #[cfg(feature = "rns-stack")] +mod voice_memo; +#[cfg(feature = "rns-stack")] mod voice_session; use std::fs; @@ -190,6 +192,9 @@ pub struct StackHandle { /// Serializes attach_live so concurrent callers cannot spawn duplicate live bridges. #[cfg(feature = "rns-stack")] attach_live_lock: Mutex<()>, + /// Opus/Ogg voice-memo encoder sessions (independent of live LXST calls). + #[cfg(feature = "rns-stack")] + voice_memo: Arc, /// Test-only: next preference/pin apply returns this error after persist (exercises rollback). #[cfg(test)] test_path_medium_apply_error: Mutex>, @@ -314,6 +319,7 @@ impl StackHandle { path_medium_op_lock: Mutex::new(()), live: std::sync::OnceLock::new(), attach_live_lock: Mutex::new(()), + voice_memo: Arc::new(voice_memo::VoiceMemoManager::new()), #[cfg(test)] test_path_medium_apply_error: Mutex::new(None), }; @@ -3025,6 +3031,71 @@ impl StackHandle { serde_json::json!({ "ok": false, "error": "voice requires live rns-stack sidecar" }) } + pub fn voice_memo_start(&self) -> serde_json::Value { + #[cfg(feature = "rns-stack")] + { + match self.voice_memo.start() { + Ok(v) => v, + Err(e) => serde_json::json!({ "ok": false, "error": e }), + } + } + #[cfg(not(feature = "rns-stack"))] + serde_json::json!({ "ok": false, "error": "voice_memo requires rns-stack sidecar" }) + } + + pub fn voice_memo_audio( + &self, + session_id: &str, + channels: u8, + samples_b64: &str, + ) -> serde_json::Value { + #[cfg(feature = "rns-stack")] + { + match self + .voice_memo + .push_audio(session_id, channels, samples_b64) + { + Ok(v) => v, + Err(e) => serde_json::json!({ "ok": false, "error": e }), + } + } + #[cfg(not(feature = "rns-stack"))] + { + let _ = (session_id, channels, samples_b64); + serde_json::json!({ "ok": false, "error": "voice_memo requires rns-stack sidecar" }) + } + } + + pub fn voice_memo_stop(&self, session_id: &str) -> serde_json::Value { + #[cfg(feature = "rns-stack")] + { + match self.voice_memo.stop(session_id) { + Ok(v) => v, + Err(e) => serde_json::json!({ "ok": false, "error": e }), + } + } + #[cfg(not(feature = "rns-stack"))] + { + let _ = session_id; + serde_json::json!({ "ok": false, "error": "voice_memo requires rns-stack sidecar" }) + } + } + + pub fn voice_memo_cancel(&self, session_id: &str) -> serde_json::Value { + #[cfg(feature = "rns-stack")] + { + match self.voice_memo.cancel(session_id) { + Ok(v) => v, + Err(e) => serde_json::json!({ "ok": false, "error": e }), + } + } + #[cfg(not(feature = "rns-stack"))] + { + let _ = session_id; + serde_json::json!({ "ok": false, "error": "voice_memo requires rns-stack sidecar" }) + } + } + pub async fn games_status(&self) -> serde_json::Value { #[cfg(feature = "rns-stack")] if let Some(live) = self.live.get() { diff --git a/reticulum-sidecar/src/stack/persistence.rs b/reticulum-sidecar/src/stack/persistence.rs index ab6a44ebe..1e8aca26f 100644 --- a/reticulum-sidecar/src/stack/persistence.rs +++ b/reticulum-sidecar/src/stack/persistence.rs @@ -1060,6 +1060,7 @@ mod tests { reply_to_hash: None, reply_to_id: None, reply_preview_text: None, + audio: None, }) .expect("send"); assert_eq!(payload["to_hash"], dest); diff --git a/reticulum-sidecar/src/stack/types.rs b/reticulum-sidecar/src/stack/types.rs index 287fdea73..8524b95c1 100644 --- a/reticulum-sidecar/src/stack/types.rs +++ b/reticulum-sidecar/src/stack/types.rs @@ -283,6 +283,14 @@ pub struct AddInterfaceRequest { pub extra_config: HashMap, } +/// Native LXMF `FIELD_AUDIO` payload for chat voice memos (Ratspeak parity). +#[derive(Debug, Clone, Deserialize)] +pub struct LxmfAudioRequest { + /// LXMF audio mode (`AM_OPUS_OGG` = 0x10). + pub mode: u8, + pub data_base64: String, +} + #[derive(Debug, Clone, Deserialize)] pub struct LxmfSendRequest { pub destination_hash: String, @@ -294,6 +302,9 @@ pub struct LxmfSendRequest { /// Optional UTF-8 quote snippet for LXMF `FIELD_REPLY_QUOTE` (0x31). #[serde(default)] pub reply_preview_text: Option, + /// Optional native LXMF audio field (Ogg/Opus voice memo). + #[serde(default)] + pub audio: Option, } /// Create an encrypted `lxm://` paper URI (no network send). diff --git a/reticulum-sidecar/src/stack/voice_memo.rs b/reticulum-sidecar/src/stack/voice_memo.rs new file mode 100644 index 000000000..263cefc03 --- /dev/null +++ b/reticulum-sidecar/src/stack/voice_memo.rs @@ -0,0 +1,345 @@ +//! LXMF voice-memo encode: PCM → Opus (`QualityMedium`) → Ogg container. +//! +//! Wire parity with Ratspeak: `AM_OPUS_OGG`, OpusHead 1ch/24 kHz, vendor `Ratspeak`, +//! 60 ms packets. Cap ~240 KiB so memos stay under the 256 KiB LXMF field limit and +//! default PN deposit size. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::Engine as _; +use lxst_core::{OpusEncoderState, Profile, RawAudioFrame}; +use ogg::{PacketWriteEndInfo, PacketWriter}; +use uuid::Uuid; + +/// Soft cap under lxmf-core 256 KiB field + default 256 KB PN deposit. +pub const VOICE_MEMO_MAX_OGG_BYTES: usize = 240 * 1024; +/// ~4 minutes of 60 ms QualityMedium frames. +pub const VOICE_MEMO_MAX_FRAME_COUNT: usize = 4_000; +const FRAME_MS: u64 = 60; +const SAMPLE_RATE_HZ: u32 = 24_000; +const CHANNELS: u8 = 1; +const SAMPLES_PER_FRAME: usize = (SAMPLE_RATE_HZ as usize * FRAME_MS as usize) / 1000; +/// QualityMedium ~8 kbps → ~60 B/packet; allow headroom. +const MAX_OPUS_PACKET_BYTES: usize = 60; + +pub struct VoiceMemoManager { + sessions: Mutex>, +} + +struct VoiceMemoSession { + encoder: OpusEncoderState, + opus_packets: Vec>, + started_ms: u64, +} + +impl Default for VoiceMemoManager { + fn default() -> Self { + Self::new() + } +} + +impl VoiceMemoManager { + pub fn new() -> Self { + Self { + sessions: Mutex::new(HashMap::new()), + } + } + + pub fn start(&self) -> Result { + let encoder = OpusEncoderState::new(Profile::QualityMedium) + .map_err(|e| format!("voice_memo_encoder: {e}"))?; + let session_id = Uuid::new_v4().to_string(); + let started_ms = now_ms(); + let mut guard = self + .sessions + .lock() + .map_err(|_| "voice_memo_lock".to_string())?; + // Bound concurrent memo sessions (one per typical UI). + if guard.len() >= 4 { + return Err("voice_memo_busy".into()); + } + guard.insert( + session_id.clone(), + VoiceMemoSession { + encoder, + opus_packets: Vec::new(), + started_ms, + }, + ); + Ok(serde_json::json!({ + "ok": true, + "session_id": session_id, + })) + } + + pub fn push_audio( + &self, + session_id: &str, + channels: u8, + samples_b64: &str, + ) -> Result { + if channels != CHANNELS { + return Err(format!("invalid_channels: expected {CHANNELS}")); + } + let samples = decode_f32_le_base64(samples_b64)?; + if samples.len() != SAMPLES_PER_FRAME { + return Err(format!( + "invalid_frame_samples: {} (expected {SAMPLES_PER_FRAME})", + samples.len() + )); + } + let mut guard = self + .sessions + .lock() + .map_err(|_| "voice_memo_lock".to_string())?; + let session = guard + .get_mut(session_id) + .ok_or_else(|| "voice_memo_session_unknown".to_string())?; + if session.opus_packets.len() >= VOICE_MEMO_MAX_FRAME_COUNT { + return Err("voice_memo_too_long".into()); + } + let frame = + RawAudioFrame::new(CHANNELS, samples).map_err(|e| format!("voice_memo_frame: {e}"))?; + let encoded = session + .encoder + .encode_frame(&frame) + .map_err(|e| format!("voice_memo_encode: {e}"))?; + if encoded.payload.len() > MAX_OPUS_PACKET_BYTES { + return Err(format!( + "voice_memo_packet_too_large: {}", + encoded.payload.len() + )); + } + session.opus_packets.push(encoded.payload); + // Rough size check: Ogg overhead ~28 B/page + headers. + let approx = session.opus_packets.iter().map(Vec::len).sum::() + 512; + if approx > VOICE_MEMO_MAX_OGG_BYTES { + return Err("voice_memo_too_large".into()); + } + Ok(serde_json::json!({ + "ok": true, + "frames": session.opus_packets.len(), + "duration_ms": session.opus_packets.len() as u64 * FRAME_MS, + })) + } + + pub fn stop(&self, session_id: &str) -> Result { + let mut guard = self + .sessions + .lock() + .map_err(|_| "voice_memo_lock".to_string())?; + let session = guard + .remove(session_id) + .ok_or_else(|| "voice_memo_session_unknown".to_string())?; + if session.opus_packets.is_empty() { + return Err("voice_memo_empty".into()); + } + let duration_ms = session.opus_packets.len() as u64 * FRAME_MS; + let ogg = mux_opus_ogg(&session.opus_packets)?; + if ogg.len() > VOICE_MEMO_MAX_OGG_BYTES { + return Err(format!( + "voice_memo_too_large: {} > {VOICE_MEMO_MAX_OGG_BYTES}", + ogg.len() + )); + } + Ok(serde_json::json!({ + "ok": true, + "session_id": session_id, + "ogg_base64": base64::engine::general_purpose::STANDARD.encode(&ogg), + "duration_ms": duration_ms, + "size_bytes": ogg.len(), + "started_ms": session.started_ms, + "mode": lxmf_core::constants::AM_OPUS_OGG, + })) + } + + pub fn cancel(&self, session_id: &str) -> Result { + let mut guard = self + .sessions + .lock() + .map_err(|_| "voice_memo_lock".to_string())?; + let removed = guard.remove(session_id).is_some(); + Ok(serde_json::json!({ + "ok": true, + "cancelled": removed, + })) + } +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +fn decode_f32_le_base64(samples_b64: &str) -> Result, String> { + let bytes = base64::engine::general_purpose::STANDARD + .decode(samples_b64.as_bytes()) + .map_err(|e| format!("samples_b64_decode: {e}"))?; + if bytes.len() % 4 != 0 { + return Err("samples_b64_misaligned".into()); + } + let mut samples = Vec::with_capacity(bytes.len() / 4); + for chunk in bytes.chunks_exact(4) { + samples.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + Ok(samples) +} + +/// RFC 7845 Ogg Opus mux: OpusHead + OpusTags (vendor Ratspeak) + audio pages. +pub fn mux_opus_ogg(opus_packets: &[Vec]) -> Result, String> { + if opus_packets.is_empty() { + return Err("voice_memo_empty".into()); + } + let mut out = Vec::new(); + { + let mut writer = PacketWriter::new(&mut out); + let head = opus_head_packet(); + writer + .write_packet(head, 0x1000_0001, PacketWriteEndInfo::EndPage, 0) + .map_err(|e| format!("ogg_write_head: {e}"))?; + let tags = opus_tags_packet(); + writer + .write_packet(tags, 0x1000_0001, PacketWriteEndInfo::EndPage, 0) + .map_err(|e| format!("ogg_write_tags: {e}"))?; + let granule_per_packet = (SAMPLE_RATE_HZ as u64 * FRAME_MS) / 1000; + let last = opus_packets.len() - 1; + for (i, packet) in opus_packets.iter().enumerate() { + let granule = granule_per_packet * (i as u64 + 1); + let end = if i == last { + PacketWriteEndInfo::EndStream + } else { + PacketWriteEndInfo::NormalPacket + }; + writer + .write_packet(packet.clone(), 0x1000_0001, end, granule) + .map_err(|e| format!("ogg_write_audio: {e}"))?; + } + } + if !out.starts_with(b"OggS") { + return Err("ogg_missing_magic".into()); + } + Ok(out) +} + +fn opus_head_packet() -> Vec { + // RFC 7845 §5.1 OpusHead + let mut head = Vec::with_capacity(19); + head.extend_from_slice(b"OpusHead"); + head.push(1); // version + head.push(CHANNELS); + head.extend_from_slice(&0u16.to_le_bytes()); // pre-skip + head.extend_from_slice(&SAMPLE_RATE_HZ.to_le_bytes()); + head.extend_from_slice(&0i16.to_le_bytes()); // output gain + head.push(0); // channel mapping family + head +} + +fn opus_tags_packet() -> Vec { + // RFC 7845 §5.2 OpusTags — vendor "Ratspeak", 0 comments + let vendor = b"Ratspeak"; + let mut tags = Vec::with_capacity(8 + 4 + vendor.len() + 4); + tags.extend_from_slice(b"OpusTags"); + tags.extend_from_slice(&(vendor.len() as u32).to_le_bytes()); + tags.extend_from_slice(vendor); + tags.extend_from_slice(&0u32.to_le_bytes()); // user comment list length + tags +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mux_opus_ogg_writes_head_and_magic() { + let packets = vec![vec![0u8; 8], vec![1u8; 10]]; + let ogg = mux_opus_ogg(&packets).expect("mux"); + assert!(ogg.starts_with(b"OggS")); + assert!(ogg.windows(8).any(|w| w == b"OpusHead")); + assert!(ogg.windows(8).any(|w| w == b"OpusTags")); + assert!(ogg.windows(8).any(|w| w == b"Ratspeak")); + // OpusHead: 1 channel, 24 kHz + let head_at = ogg.windows(8).position(|w| w == b"OpusHead").expect("head"); + assert_eq!(ogg[head_at + 9], 1); // channels + let rate = u32::from_le_bytes([ + ogg[head_at + 12], + ogg[head_at + 13], + ogg[head_at + 14], + ogg[head_at + 15], + ]); + assert_eq!(rate, 24_000); + } + + #[test] + fn memo_encode_silent_frames_round_trip_size() { + let mgr = VoiceMemoManager::new(); + let start = mgr.start().expect("start"); + let session_id = start["session_id"].as_str().expect("id").to_string(); + let silence = vec![0f32; SAMPLES_PER_FRAME]; + let mut bytes = Vec::with_capacity(silence.len() * 4); + for s in silence { + bytes.extend_from_slice(&s.to_le_bytes()); + } + let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes); + for _ in 0..5 { + mgr.push_audio(&session_id, 1, &b64).expect("audio"); + } + let stop = mgr.stop(&session_id).expect("stop"); + assert_eq!(stop["ok"], true); + assert_eq!(stop["duration_ms"], 300); + let ogg = base64::engine::general_purpose::STANDARD + .decode(stop["ogg_base64"].as_str().expect("b64")) + .expect("decode"); + assert!(ogg.starts_with(b"OggS")); + assert!(ogg.len() <= VOICE_MEMO_MAX_OGG_BYTES); + assert!(ogg.len() > 40); + } + + #[test] + fn cancel_then_stop_fails() { + let mgr = VoiceMemoManager::new(); + let start = mgr.start().expect("start"); + let session_id = start["session_id"].as_str().expect("id").to_string(); + mgr.cancel(&session_id).expect("cancel"); + let err = mgr.stop(&session_id).expect_err("stop after cancel"); + assert!(err.contains("unknown"), "{err}"); + } + + #[test] + fn encode_then_lxmf_audio_field_round_trips() { + use base64::Engine as _; + use lxmf_core::constants::AM_OPUS_OGG; + use lxmf_core::constants::DeliveryMethod; + use lxmf_core::message::LxMessage; + + let mgr = VoiceMemoManager::new(); + let start = mgr.start().expect("start"); + let session_id = start["session_id"].as_str().expect("id").to_string(); + let silence = vec![0f32; SAMPLES_PER_FRAME]; + let mut bytes = Vec::with_capacity(silence.len() * 4); + for s in silence { + bytes.extend_from_slice(&s.to_le_bytes()); + } + let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes); + mgr.push_audio(&session_id, 1, &b64).expect("audio"); + let stop = mgr.stop(&session_id).expect("stop"); + let ogg = base64::engine::general_purpose::STANDARD + .decode(stop["ogg_base64"].as_str().expect("b64")) + .expect("decode"); + + let mut msg = LxMessage::new( + [0u8; 16], + [1u8; 16], + "", + "[voice:60]", + DeliveryMethod::Direct, + ); + msg.set_audio_field(AM_OPUS_OGG, &ogg).expect("set"); + let audio = msg.audio_field().expect("field").expect("some"); + assert_eq!(audio.mode, AM_OPUS_OGG); + assert_eq!(audio.bytes, ogg.as_slice()); + } +} diff --git a/src/main/index.ts b/src/main/index.ts index 59fa36b97..66b9a6b8a 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -158,6 +158,10 @@ import { handleNobleBleToRadioWrite } from './noble-ble-ipc'; import { NobleBleManager, type NobleSessionId } from './noble-ble-manager'; import { readFileUpTo } from './readFileUpTo'; import { createRendererHeartbeatWatchdog } from './rendererHeartbeatWatchdog'; +import { + readReticulumAttachmentBytes, + takeReticulumAttachmentAudioRateToken, +} from './reticulum-attachment-audio'; import { readReticulumAttachmentAsDataUrl, takeReticulumAttachmentImageRateToken, @@ -4908,6 +4912,27 @@ ipcMain.handle('chat:readReticulumAttachmentAsDataUrl', async (event, opts: unkn } }); +ipcMain.handle('chat:readReticulumAttachmentBytes', async (event, filePath: unknown) => { + if (!validateIpcSender(event)) throw new Error('IPC sender validation failed'); + if (typeof filePath !== 'string' || !filePath.trim() || filePath.length > 512) { + throw new Error('filePath must be a non-empty string'); + } + if (!takeReticulumAttachmentAudioRateToken()) { + console.debug('[IPC] chat:readReticulumAttachmentBytes rate limited'); + return { dataBase64: null }; + } + try { + const dataBase64 = await readReticulumAttachmentBytes(filePath); + return { dataBase64 }; + } catch (err) { + console.error( + '[IPC] chat:readReticulumAttachmentBytes failed:', + sanitizeLogMessage(err instanceof Error ? err.message : String(err)), + ); + throw err; + } +}); + ipcMain.handle('meshtastic:xmodemPickUpload', async (event) => { if (!validateIpcSender(event)) throw new Error('IPC sender validation failed'); if (!mainWindow) return null; diff --git a/src/main/ipc/reticulum-handlers.ts b/src/main/ipc/reticulum-handlers.ts index fb6462b90..45eb0b39f 100644 --- a/src/main/ipc/reticulum-handlers.ts +++ b/src/main/ipc/reticulum-handlers.ts @@ -6,6 +6,15 @@ import type { ReticulumSidecarStartOptions, ReticulumSidecarStatus, } from '../../shared/reticulum-types'; +import { + isVoiceMemoApiPath, + parseVoiceMemoAudioRequest, + parseVoiceMemoSessionRequest, + VOICE_MEMO_AUDIO_API_PATH, + VOICE_MEMO_CANCEL_API_PATH, + VOICE_MEMO_START_API_PATH, + VOICE_MEMO_STOP_API_PATH, +} from '../../shared/reticulum-voice-memo-types'; import { canonicalizeReticulumDestinationHash } from '../../shared/reticulumDestinationHash'; import { isExpectedReticulumProxyError, @@ -259,6 +268,9 @@ export function registerReticulumIpcHandlers(deps: ReticulumIpcDeps): void { if (isVoiceAudioApiPath(pathArg)) { throw new Error('voice PCM ingest requires reticulum:voiceSendAudio'); } + if (isVoiceMemoApiPath(pathArg)) { + throw new Error('voice memo requires reticulum:voiceMemo* IPC channels'); + } if (isGamesApiPath(pathArg)) { throw new Error('LRGP games require reticulum:games* IPC channels'); } @@ -272,6 +284,17 @@ export function registerReticulumIpcHandlers(deps: ReticulumIpcDeps): void { } }); + /** + * LXMF voice memo PCM ingest. Separate bucket from live LXST voice so memo + * recording does not starve live-call TX or the shared proxy ceiling. + * 2000/min matches the live-voice bucket (QualityHigh ~16.7 frames/s). + */ + const reticulumVoiceMemoAudioIpcRateLimit = createIpcRateLimiter({ + max: 2000, + windowMs: MS_PER_MINUTE, + label: 'reticulum:voiceMemoSendAudio', + }); + /** * Realtime LXST PCM frames. Uses a dedicated rate limit (not the shared 900/min * proxy ceiling) so voice TX does not starve control-plane proxy IPC. @@ -292,6 +315,64 @@ export function registerReticulumIpcHandlers(deps: ReticulumIpcDeps): void { } }); + ipcMain.handle('reticulum:voiceMemoStart', async (event, opts: unknown) => { + assertIpcSender(event, 'reticulum:voiceMemoStart'); + const body = opts != null && typeof opts === 'object' && !Array.isArray(opts) ? opts : {}; + try { + const m = ensureManager(); + return await m.proxyPost(VOICE_MEMO_START_API_PATH, body); + } catch (err) { + // catch-no-log-ok settleReticulumProxyFailure logs expected failures / rethrows unexpected + return settleReticulumProxyFailure('voiceMemoStart', err, VOICE_MEMO_START_API_PATH); + } + }); + + ipcMain.handle('reticulum:voiceMemoSendAudio', async (event, opts: unknown) => { + assertIpcSender(event, 'reticulum:voiceMemoSendAudio'); + reticulumVoiceMemoAudioIpcRateLimit.checkOrThrow(); + const parsed = parseVoiceMemoAudioRequest(opts); + if ('error' in parsed) { + return { ok: false, error: parsed.error }; + } + try { + const m = ensureManager(); + return await m.proxyPost(VOICE_MEMO_AUDIO_API_PATH, parsed); + } catch (err) { + // catch-no-log-ok settleReticulumProxyFailure logs expected failures / rethrows unexpected + return settleReticulumProxyFailure('voiceMemoSendAudio', err, VOICE_MEMO_AUDIO_API_PATH); + } + }); + + ipcMain.handle('reticulum:voiceMemoStop', async (event, opts: unknown) => { + assertIpcSender(event, 'reticulum:voiceMemoStop'); + const parsed = parseVoiceMemoSessionRequest(opts); + if ('error' in parsed) { + return { ok: false, error: parsed.error }; + } + try { + const m = ensureManager(); + return await m.proxyPost(VOICE_MEMO_STOP_API_PATH, parsed); + } catch (err) { + // catch-no-log-ok settleReticulumProxyFailure logs expected failures / rethrows unexpected + return settleReticulumProxyFailure('voiceMemoStop', err, VOICE_MEMO_STOP_API_PATH); + } + }); + + ipcMain.handle('reticulum:voiceMemoCancel', async (event, opts: unknown) => { + assertIpcSender(event, 'reticulum:voiceMemoCancel'); + const parsed = parseVoiceMemoSessionRequest(opts); + if ('error' in parsed) { + return { ok: false, error: parsed.error }; + } + try { + const m = ensureManager(); + return await m.proxyPost(VOICE_MEMO_CANCEL_API_PATH, parsed); + } catch (err) { + // catch-no-log-ok settleReticulumProxyFailure logs expected failures / rethrows unexpected + return settleReticulumProxyFailure('voiceMemoCancel', err, VOICE_MEMO_CANCEL_API_PATH); + } + }); + ipcMain.handle('reticulum:factoryReset', async (event) => { assertIpcSender(event, 'reticulum:factoryReset'); try { diff --git a/src/main/ipc/reticulum-proxy-rate-limit.contract.test.ts b/src/main/ipc/reticulum-proxy-rate-limit.contract.test.ts index ce069f273..e73c2614f 100644 --- a/src/main/ipc/reticulum-proxy-rate-limit.contract.test.ts +++ b/src/main/ipc/reticulum-proxy-rate-limit.contract.test.ts @@ -65,7 +65,7 @@ describe('reticulum proxy rate limit + 100k peer ceilings (source contract)', () ] as const) { const handleIdx = HANDLERS_SOURCE.indexOf(`ipcMain.handle('${channel}'`); expect(handleIdx, channel).toBeGreaterThanOrEqual(0); - const afterHandle = HANDLERS_SOURCE.slice(handleIdx, handleIdx + 900); + const afterHandle = HANDLERS_SOURCE.slice(handleIdx, handleIdx + 1200); const tryIdx = afterHandle.indexOf('try {'); const checkIdx = afterHandle.indexOf('reticulumProxyIpcRateLimit.checkOrThrow()'); expect(tryIdx, channel).toBeGreaterThanOrEqual(0); @@ -86,6 +86,20 @@ describe('reticulum proxy rate limit + 100k peer ceilings (source contract)', () expect(preload).not.toMatch(/invoke\('reticulum:proxyPost',\s*'\/api\/v1\/voice\/audio'/); }); + it('routes voice memo PCM through dedicated IPC with its own rate limit and blocks proxy', () => { + expect(HANDLERS_SOURCE).toContain("ipcMain.handle('reticulum:voiceMemoStart'"); + expect(HANDLERS_SOURCE).toContain("ipcMain.handle('reticulum:voiceMemoSendAudio'"); + expect(HANDLERS_SOURCE).toContain("ipcMain.handle('reticulum:voiceMemoStop'"); + expect(HANDLERS_SOURCE).toContain("ipcMain.handle('reticulum:voiceMemoCancel'"); + expect(HANDLERS_SOURCE).toContain('voice memo requires reticulum:voiceMemo* IPC channels'); + const preload = readFileSync(join(__dirname, '../../preload/index.ts'), 'utf-8'); + expect(preload).toContain("ipcRenderer.invoke('reticulum:voiceMemoStart'"); + expect(preload).toContain("ipcRenderer.invoke('reticulum:voiceMemoSendAudio'"); + expect(preload).toContain("ipcRenderer.invoke('reticulum:voiceMemoStop'"); + expect(preload).toContain("ipcRenderer.invoke('reticulum:voiceMemoCancel'"); + expect(preload).not.toMatch(/invoke\('reticulum:proxyPost',\s*'\/api\/v1\/voice\/memo/); + }); + it('routes LRGP games through dedicated IPC with its own rate limit', () => { expect(HANDLERS_SOURCE).toMatch( /const reticulumGamesIpcRateLimit = createIpcRateLimiter\(\{\s*max:\s*600,[\s\S]*?label:\s*'reticulum:games'/, diff --git a/src/main/reticulum-attachment-audio.ts b/src/main/reticulum-attachment-audio.ts new file mode 100644 index 000000000..667be44c8 --- /dev/null +++ b/src/main/reticulum-attachment-audio.ts @@ -0,0 +1,64 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { isUnderCanonicalRoot } from './pathCanonical'; +import { readFileUpTo } from './readFileUpTo'; +import { + assertReticulumAttachmentPathJailed, + getReticulumAttachmentsDir, +} from './reticulum-attachment-path'; + +/** Maximum audio attachment size for inline read (256 KiB). */ +export const RETICULUM_ATTACHMENT_AUDIO_MAX_BYTES = 256 * 1024; + +/** IPC rate limit — max reads per rolling window. */ +export const RETICULUM_ATTACHMENT_AUDIO_RATE_LIMIT_MAX = 60; +export const RETICULUM_ATTACHMENT_AUDIO_RATE_LIMIT_WINDOW_MS = 60_000; + +/** OggS magic bytes for basic file-type validation (prevents non-Ogg reads). */ +const OGG_MAGIC = Buffer.from([0x4f, 0x67, 0x67, 0x53]); + +const audioRateTimestamps: number[] = []; + +/** Returns true when a new attachment audio read is allowed under the IPC rate limit. */ +export function takeReticulumAttachmentAudioRateToken(now = Date.now()): boolean { + const cutoff = now - RETICULUM_ATTACHMENT_AUDIO_RATE_LIMIT_WINDOW_MS; + while (audioRateTimestamps.length > 0) { + const oldest = audioRateTimestamps[0]; + if (oldest === undefined || oldest >= cutoff) break; + audioRateTimestamps.shift(); + } + if (audioRateTimestamps.length >= RETICULUM_ATTACHMENT_AUDIO_RATE_LIMIT_MAX) { + return false; + } + audioRateTimestamps.push(now); + return true; +} + +/** Test helper — clears the rate-limit window. */ +export function resetReticulumAttachmentAudioRateLimitForTests(): void { + audioRateTimestamps.length = 0; +} + +/** + * Read a jailed Reticulum audio attachment (OggS) and return its bytes as base64. + * Validates magic bytes so non-Ogg files are rejected before the bytes leave main. + * Returns null when the file is empty, too large, not OggS, or out of jail. + */ +export async function readReticulumAttachmentBytes(filePath: string): Promise { + const jailed = assertReticulumAttachmentPathJailed(filePath); + const realPath = await fs.realpath(path.resolve(jailed)); + if (!isUnderCanonicalRoot(realPath, getReticulumAttachmentsDir())) { + throw new Error('audio attachment path outside reticulum attachments directory'); + } + const stat = await fs.stat(realPath); + if (!stat.isFile()) { + throw new Error('audio attachment path is not a file'); + } + const buf = await readFileUpTo(realPath, RETICULUM_ATTACHMENT_AUDIO_MAX_BYTES); + if (buf.length === 0) return null; + if (buf.length < 4 || !buf.subarray(0, 4).equals(OGG_MAGIC)) { + return null; + } + return buf.toString('base64'); +} diff --git a/src/preload/index.ts b/src/preload/index.ts index def912186..fab97c0c2 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -17,6 +17,7 @@ import type { OutboxStatus, ReadReticulumAttachmentAsDataUrlOpts, ReadReticulumAttachmentAsDataUrlResult, + ReadReticulumAttachmentBytesResult, ReticulumIdentityBackupImportDialogResult, ReticulumIdentityExportSaveResult, ReticulumIdentityImportDialogResult, @@ -1225,6 +1226,15 @@ contextBridge.exposeInMainWorld('electronAPI', { sendAudio: (opts: { profile?: number; channels: number; samples_b64: string }) => unwrapReticulumProxy(ipcRenderer.invoke('reticulum:voiceSendAudio', opts)), }, + voiceMemo: { + start: () => unwrapReticulumProxy(ipcRenderer.invoke('reticulum:voiceMemoStart', {})), + sendAudio: (opts: { session_id: string; channels: 1; samples_b64: string }) => + unwrapReticulumProxy(ipcRenderer.invoke('reticulum:voiceMemoSendAudio', opts)), + stop: (opts: { session_id: string }) => + unwrapReticulumProxy(ipcRenderer.invoke('reticulum:voiceMemoStop', opts)), + cancel: (opts: { session_id: string }) => + unwrapReticulumProxy(ipcRenderer.invoke('reticulum:voiceMemoCancel', opts)), + }, /** LRGP games — dedicated IPC (blocked on generic proxy). */ games: { getStatus: () => unwrapReticulumProxy(ipcRenderer.invoke('reticulum:gamesStatus')), @@ -1337,6 +1347,11 @@ contextBridge.exposeInMainWorld('electronAPI', { 'chat:readReticulumAttachmentAsDataUrl', opts, ) as Promise, + readReticulumAttachmentBytes: (filePath: string) => + ipcRenderer.invoke( + 'chat:readReticulumAttachmentBytes', + filePath, + ) as Promise, linkPreview: { fetch: (url: string) => ipcRenderer.invoke('chat:fetchLinkPreview', url) as Promise<{ diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 72cb0a7c2..d35c21856 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -243,6 +243,8 @@ import { openReticulumGameSession } from './lib/reticulum/reticulumGamesSession' import { setReticulumManualStackStopSuppress } from './lib/reticulum/reticulumManualStackStopSuppress'; import { resolveReticulumSelfHeaderLabel } from './lib/reticulum/reticulumSelfNodeLabel'; import { skipReticulumStartupAutostartGate } from './lib/reticulum/reticulumStartupAutostartGate'; +import { startReticulumVoiceMemo } from './lib/reticulum/reticulumVoiceMemo'; +import { sendReticulumVoiceMemo } from './lib/reticulum/sendReticulumVoiceMemo'; import { logRfReconnectFailure, reconnectRfFromLastConnection } from './lib/rfReconnectHelper'; import { scheduleReticulumVacuumIfNeeded } from './lib/startupDbPrune'; import { getStoredMeshProtocol, MESH_PROTOCOL_STORAGE_KEY } from './lib/storedMeshProtocol'; @@ -280,6 +282,7 @@ import { usePositionHistoryStore } from './stores/positionHistoryStore'; import { useReticulumGamesStore } from './stores/reticulumGamesStore'; import { useReticulumIdentityStore } from './stores/reticulumIdentityStore'; import { useReticulumPeerStore } from './stores/reticulumPeerStore'; +import { useReticulumVoiceMemoStore } from './stores/reticulumVoiceMemoStore'; import { useRncpTransferStore } from './stores/rncpTransferStore'; import { useRrcSessionStore } from './stores/rrcSessionStore'; import { useTimeFormatStore } from './stores/timeFormatStore'; @@ -3200,6 +3203,49 @@ function AppContent() { dmOnlyChat={capabilities.hasReticulumInterfaceConfig} hasRncpTransfer={capabilities.hasRncpTransfer} hasLxstVoice={capabilities.hasLxstVoice} + hasReticulumVoiceMemo={capabilities.hasReticulumVoiceMemo} + onVoiceMemo={ + capabilities.hasReticulumVoiceMemo && reticulumIdentityId + ? (destination: number) => { + const phase = useReticulumVoiceMemoStore.getState().phase; + if (phase === 'recording' || phase === 'stopping') { + sendReticulumVoiceMemo({ + identityId: reticulumIdentityId, + destination, + onOversize: () => { + addToast( + t('chatPanel.voiceMemo.tooLargeForWire'), + 'warning', + ); + }, + onNoPropagationNode: () => { + addToast( + t('chatPanel.reticulumNoPropagationNode'), + 'error', + ); + }, + onTooLargeForPropagation: () => { + addToast( + t('chatPanel.voiceMemo.tooLargeForPropagation'), + 'info', + ); + }, + }); + return; + } + void startReticulumVoiceMemo().then((ok) => { + if (!ok) { + const err = useReticulumVoiceMemoStore.getState().lastError; + if (err === 'call_busy') { + addToast(t('chatPanel.voiceMemo.callBusy'), 'warning'); + } else if (err === 'mic_denied') { + addToast(t('chatPanel.voiceMemo.micDenied'), 'error'); + } + } + }); + } + : undefined + } hasLrgpGames={capabilities.hasLrgpGames} hasLxmfPaper={capabilities.hasLxmfPaper} showLxmfDeliveryStatus={capabilities.hasLxmfDeliveryStatus} diff --git a/src/renderer/components/ChatComposer.tsx b/src/renderer/components/ChatComposer.tsx index 0c588fdfa..87e644de4 100644 --- a/src/renderer/components/ChatComposer.tsx +++ b/src/renderer/components/ChatComposer.tsx @@ -1,7 +1,7 @@ /* eslint-disable react-hooks/refs */ import 'emoji-picker-element'; -import { ChevronDown, ChevronUp, CornerUpLeft, MapPin } from 'lucide-react-motion'; +import { ChevronDown, ChevronUp, CornerUpLeft, MapPin, Mic } from 'lucide-react-motion'; import { type ReactNode, type RefObject, @@ -19,6 +19,7 @@ import { errLikeToLogString } from '@/renderer/lib/errLikeToLogString'; import { useIconTrigger } from '@/renderer/lib/icons/iconMotionContext'; import { nodeDisplayName } from '@/renderer/lib/nodeLongNameOrHex'; import type { ChatMessage, MeshNode, MeshProtocol } from '@/renderer/lib/types'; +import { useReticulumVoiceMemoStore } from '@/renderer/stores/reticulumVoiceMemoStore'; import type { OutboxEntry, OutboxEntryInput } from '@/shared/electron-api.types'; import { @@ -191,6 +192,8 @@ export interface ChatComposerProps { */ onSendLocationWaypoint?: (lat: number, lon: number) => Promise; textareaRef?: RefObject; + /** When set, renders a mic button that triggers voice memo recording. */ + onVoiceMemo?: () => void; className?: string; } @@ -224,6 +227,7 @@ export function ChatComposer({ resolveShareLocation, onSendLocationWaypoint, textareaRef, + onVoiceMemo, className, }: ChatComposerProps) { const { t } = useTranslation(); @@ -1519,20 +1523,25 @@ export function ChatComposer({ : null} ) : ( - +
+ {onVoiceMemo != null && !input.trim() && !sending && ( + + )} + +
)} @@ -1589,3 +1598,41 @@ export function ChatComposer({ ); } + +function VoiceMemoComposerButton({ + onVoiceMemo, + disabled, +}: { + onVoiceMemo: () => void; + disabled?: boolean; +}) { + const { t } = useTranslation(); + const phase = useReticulumVoiceMemoStore((s) => s.phase); + const elapsedSec = useReticulumVoiceMemoStore((s) => s.elapsedSec); + const recording = phase === 'recording' || phase === 'starting'; + const busy = phase === 'stopping' || phase === 'sending'; + return ( + + ); +} diff --git a/src/renderer/components/ChatPanel.tsx b/src/renderer/components/ChatPanel.tsx index 827b4e6e2..d147f33aa 100644 --- a/src/renderer/components/ChatPanel.tsx +++ b/src/renderer/components/ChatPanel.tsx @@ -169,6 +169,7 @@ import { import { ReticulumMessageStatusBadge } from './ReticulumMessageStatusBadge'; import { ReticulumProfileIconSlot } from './ReticulumProfileIcon'; import { ReticulumPropagationNotice } from './ReticulumPropagationNotice'; +import { ReticulumVoiceMemoLine } from './ReticulumVoiceMemoLine'; import { useToast } from './Toast'; function chatPanelIsLinux(): boolean { @@ -529,6 +530,10 @@ export interface ChatPanelProps { hasRncpTransfer?: boolean; /** Reticulum: LXST voice Call control in the DM header. */ hasLxstVoice?: boolean; + /** Reticulum: LXMF voice memo mic button in composer + playback line in chat. */ + hasReticulumVoiceMemo?: boolean; + /** Called when the user presses the mic button (destination = active DM node). */ + onVoiceMemo?: (destination: number) => void; /** Reticulum: LRGP games Challenge control in the DM header. */ hasLrgpGames?: boolean; /** Reticulum: LXMF paper Share as paper / Scan paper controls. */ @@ -588,6 +593,8 @@ function ChatPanel({ reticulumStackLive = false, hasRncpTransfer = false, hasLxstVoice = false, + hasReticulumVoiceMemo = false, + onVoiceMemo, hasLrgpGames = false, hasLxmfPaper = false, resolveShareLocation, @@ -2859,8 +2866,16 @@ function ChatPanel({ {/* Message text with optional search highlight (div: ChatPayloadText may render block link previews) */}
- {showLxmfAttachmentLine && - parseReticulumAttachmentPayload(msg.payload) ? ( + {hasReticulumVoiceMemo && + msg.reticulumAttachmentKind === 'audio' && + msg.reticulumAttachmentPath ? ( + + ) : showLxmfAttachmentLine && + parseReticulumAttachmentPayload(msg.payload) ? ( { + if (activeDmNode == null) return; + onVoiceMemo(activeDmNode); + } + : undefined + } /> {chatActionError?.viewKey === viewKey && ( diff --git a/src/renderer/components/ReticulumMessageStatusBadge.test.tsx b/src/renderer/components/ReticulumMessageStatusBadge.test.tsx index 7d08de34e..b123d5767 100644 --- a/src/renderer/components/ReticulumMessageStatusBadge.test.tsx +++ b/src/renderer/components/ReticulumMessageStatusBadge.test.tsx @@ -95,14 +95,17 @@ describe('ReticulumMessageStatusBadge', () => { expect(screen.getByText(/reticulumPnAbbrev\s+\u{1F3E0}/u)).toBeTruthy(); }); - it('shows red X (not house) for failed stored_locally', async () => { + it('shows Direct-only notice for message_too_large_for_propagation', async () => { await renderAndAssertAxe( - , + , ); expect( - screen.getByLabelText('chatPanel.sentViaLocalPropagation: chatPanel.reticulumSendFailed'), + screen.getByLabelText('chatPanel.sentViaPropagation: chatPanel.voiceMemo.directOnlyBadge'), ).toBeTruthy(); - expect(screen.getByText(/reticulumPnAbbrev\s+\u2717/)).toBeTruthy(); - expect(screen.queryByText(/\u{1F3E0}/u)).toBeNull(); }); }); diff --git a/src/renderer/components/ReticulumMessageStatusBadge.tsx b/src/renderer/components/ReticulumMessageStatusBadge.tsx index f3ff6c8c2..7725f173d 100644 --- a/src/renderer/components/ReticulumMessageStatusBadge.tsx +++ b/src/renderer/components/ReticulumMessageStatusBadge.tsx @@ -58,10 +58,14 @@ function statusIcon( function statusColorClass( status: OutboundStatus, deliveryMethod: MessageRecord['reticulumDeliveryMethod'] | undefined, + error?: string, ): string { if (deliveryMethod === 'stored_locally' && status !== 'failed') { return 'text-amber-400'; } + if (status === 'failed' && error === 'message_too_large_for_propagation') { + return 'text-amber-400'; + } switch (status) { case 'sending': return 'text-muted'; @@ -99,6 +103,9 @@ function statusLabelText( } return t('chatPanel.reticulumSendDelivered'); default: + if (error === 'message_too_large_for_propagation') { + return t('chatPanel.voiceMemo.directOnlyBadge'); + } return error ?? t('chatPanel.reticulumSendFailed'); } } @@ -147,7 +154,7 @@ export function ReticulumMessageStatusBadge({ ); diff --git a/src/renderer/components/ReticulumVoiceMemoLine.test.tsx b/src/renderer/components/ReticulumVoiceMemoLine.test.tsx new file mode 100644 index 000000000..2827fe4d4 --- /dev/null +++ b/src/renderer/components/ReticulumVoiceMemoLine.test.tsx @@ -0,0 +1,55 @@ +import { render, screen } from '@testing-library/react'; +import type { ReactElement } from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { axe } from 'vitest-axe'; + +import { hydrateAxeThemeColors } from '@/renderer/lib/a11yTestHelpers'; + +import { ReticulumVoiceMemoLine } from './ReticulumVoiceMemoLine'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})); + +// computeWaveformFromOgg needs decodeAudioData — stub out for unit tests +vi.mock('@/renderer/lib/reticulum/computeWaveform', () => ({ + computeWaveformFromOgg: vi.fn().mockResolvedValue({ + bars: new Array(40).fill(0.5), + durationSec: 3, + }), +})); + +const readReticulumAttachmentBytes = vi.fn(); + +beforeEach(() => { + readReticulumAttachmentBytes.mockReset(); + readReticulumAttachmentBytes.mockResolvedValue({ dataBase64: null }); + window.electronAPI = { + ...window.electronAPI, + chat: { + ...window.electronAPI?.chat, + readReticulumAttachmentBytes: (...args: unknown[]) => readReticulumAttachmentBytes(...args), + }, + }; +}); + +async function renderAxe(ui: ReactElement): Promise> { + const view = render(ui); + hydrateAxeThemeColors(view.container); + expect(await axe(view.container)).toHaveNoViolations(); + return view; +} + +describe('ReticulumVoiceMemoLine', () => { + it('renders play button and seek control', async () => { + await renderAxe( + , + ); + expect(screen.getByRole('button', { name: 'chatPanel.voiceMemo.playAria' })).toBeDefined(); + expect(screen.getByRole('slider', { name: 'chatPanel.voiceMemo.seekAria' })).toBeDefined(); + }); + + it('passes axe with no violations', async () => { + await renderAxe(); + }); +}); diff --git a/src/renderer/components/ReticulumVoiceMemoLine.tsx b/src/renderer/components/ReticulumVoiceMemoLine.tsx new file mode 100644 index 000000000..0f468ab2b --- /dev/null +++ b/src/renderer/components/ReticulumVoiceMemoLine.tsx @@ -0,0 +1,193 @@ +import { Pause, Play } from 'lucide-react-motion'; +import { useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { computeWaveformFromOgg } from '@/renderer/lib/reticulum/computeWaveform'; + +const BAR_COUNT = 40; +const BAR_MIN_HEIGHT = 2; + +export interface ReticulumVoiceMemoLineProps { + /** Local on-disk path of the jailed OggS audio file. */ + attachmentPath: string; + /** Known duration in seconds (from ingest; may be 0 before decode). */ + durationSec?: number; + /** LXMF audio mode (16 = AM_OPUS_OGG). */ + audioMode?: number; +} + +function formatDuration(sec: number): string { + const m = Math.floor(sec / 60); + const s = Math.floor(sec % 60); + return `${String(m).padStart(1, '0')}:${String(s).padStart(2, '0')}`; +} + +export function ReticulumVoiceMemoLine({ + attachmentPath, + durationSec = 0, + audioMode, +}: Readonly) { + const { t } = useTranslation(); + const [bars, setBars] = useState(new Array(BAR_COUNT).fill(0)); + const [resolvedDuration, setResolvedDuration] = useState(durationSec); + const [playing, setPlaying] = useState(false); + const [currentSec, setCurrentSec] = useState(0); + const [loadError, setLoadError] = useState(false); + const audioRef = useRef(null); + const objectUrlRef = useRef(null); + const rafRef = useRef(null); + + // Fetch bytes and decode + useEffect(() => { + let cancelled = false; + void (async () => { + try { + const res = await window.electronAPI.chat.readReticulumAttachmentBytes(attachmentPath); + if (cancelled || !res.dataBase64) return; + const waveform = await computeWaveformFromOgg(res.dataBase64, BAR_COUNT); + if (cancelled) return; + if (waveform) { + setBars(waveform.bars); + setResolvedDuration((prev) => (prev > 0 ? prev : waveform.durationSec)); + } + // Build object URL for audio element + const binary = atob(res.dataBase64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + const blob = new Blob([bytes], { type: 'audio/ogg' }); + const url = URL.createObjectURL(blob); + objectUrlRef.current = url; + if (audioRef.current) { + audioRef.current.src = url; + } + } catch { + // catch-no-log-ok: attachment may be absent or path jailed — show error state + if (!cancelled) setLoadError(true); + } + })(); + return () => { + cancelled = true; + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current); + objectUrlRef.current = null; + } + if (rafRef.current != null) cancelAnimationFrame(rafRef.current); + }; + }, [attachmentPath]); + + // RAF progress updater + const updateProgress = () => { + const audio = audioRef.current; + if (!audio) return; + setCurrentSec(audio.currentTime); + if (!audio.paused) { + rafRef.current = requestAnimationFrame(updateProgress); + } + }; + + const handlePlayPause = () => { + const audio = audioRef.current; + if (!audio || loadError) return; + if (audio.paused) { + void audio.play(); + } else { + audio.pause(); + } + }; + + const handleSeek = (e: React.MouseEvent) => { + const audio = audioRef.current; + if (!audio || loadError || !resolvedDuration) return; + const rect = e.currentTarget.getBoundingClientRect(); + const ratio = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); + audio.currentTime = ratio * resolvedDuration; + setCurrentSec(audio.currentTime); + }; + + const playedRatio = resolvedDuration > 0 ? Math.min(1, currentSec / resolvedDuration) : 0; + const displaySec = playing ? currentSec : resolvedDuration; + const modeLabel = audioMode === 16 ? 'Opus' : undefined; + + return ( +
+ {/* Playback uses Web Audio via the element; captions N/A for short voice memos. */} + {/* eslint-disable-next-line jsx-a11y/media-has-caption -- voice memo clip, not captioned media */} +