From e75bf6fc7d3ce6a25373751abbbb26e9cdd50393 Mon Sep 17 00:00:00 2001 From: sonix aka privex Date: Sun, 27 Sep 2026 17:02:32 +0530 Subject: [PATCH 1/2] fix(server): a closing WebSocket no longer cuts off the account's newer one The online map kept one sender per account and any closing connection removed it - so with two connections (two tabs, or two devices), the OLDER one closing wiped the NEWER one's live route: it looked connected but got no live messages until it reconnected. The mirror case was broken too: when the newest closed, an older connection still open never got live delivery back. The map now lists the account's open connections, newest last (RAM only, never persisted or logged - the server already accepts each socket, so it learns nothing new). A closing connection removes only itself, and live pushes go to the newest one still open, falling back past one that's already closing. Pushes still go to ONE connection, not all: in the single-mailbox model a device that can't decrypt a message would ack (delete) it before the one that can. Co-Authored-By: Claude Opus 5.5 --- server/src/ws/handler.rs | 3 +- server/src/ws/state.rs | 109 +++++++++++++++++++++++++++++++----- server/tests/integration.rs | 60 ++++++++++++++++++++ 3 files changed, 158 insertions(+), 14 deletions(-) diff --git a/server/src/ws/handler.rs b/server/src/ws/handler.rs index 0b729dd..d07da14 100644 --- a/server/src/ws/handler.rs +++ b/server/src/ws/handler.rs @@ -221,7 +221,8 @@ async fn handle_socket(socket: WebSocket, st: AppState, user_id: String) { } writer.abort(); - st.online.remove(&user_id); + // Only THIS connection: a newer one for the account must stay live. + st.online.remove(&user_id, &tx); } fn base64_content(bytes: &[u8]) -> String { diff --git a/server/src/ws/state.rs b/server/src/ws/state.rs index 488ae81..5f175c0 100644 --- a/server/src/ws/state.rs +++ b/server/src/ws/state.rs @@ -1,15 +1,22 @@ -// In-memory online map: user_id -> outbound channel. NEVER persisted. Cleared -// on disconnect. Used to push messages to currently-connected recipients. +// In-memory online map: user_id -> the account's open connections, newest last. +// NEVER persisted or logged; a connection is listed only while its socket is open. // -// ponytail: one connection per user (a reconnect replaces the previous sender). -// Multi-device fan-out lands when device linking does. +// Live pushes go to the NEWEST connection only - the single-mailbox model +// (KNOWN_LIMITATIONS): an incoming message lands on whichever device acks it +// first, so fanning it out to every connection would let one that can't decrypt +// it ack (delete) it before the one that can. When a connection closes it removes +// only ITSELF, and the newest one still open takes over live delivery - an older +// socket closing must never cut off a newer one, and the newest closing must hand +// delivery back to one that's still open. +use dashmap::mapref::entry::Entry; use dashmap::DashMap; +use tokio::sync::mpsc::error::SendError; use tokio::sync::mpsc::UnboundedSender; #[derive(Default)] pub struct Online { - map: DashMap>, + map: DashMap>>, } impl Online { @@ -17,20 +24,96 @@ impl Online { Self::default() } + /// A connection opened: it becomes the account's live-delivery target. pub fn insert(&self, user_id: &str, tx: UnboundedSender) { - self.map.insert(user_id.to_string(), tx); + self.map.entry(user_id.to_string()).or_default().push(tx); } - pub fn remove(&self, user_id: &str) { - self.map.remove(user_id); + /// A connection closed: drop just that one (the entry goes once none are left). + pub fn remove(&self, user_id: &str, tx: &UnboundedSender) { + if let Entry::Occupied(mut e) = self.map.entry(user_id.to_string()) { + e.get_mut().retain(|c| !c.same_channel(tx)); + if e.get().is_empty() { + e.remove(); + } + } } - /// Best-effort push to a connected user. Returns false if offline or the - /// channel is closed. + /// Best-effort push to the account's newest open connection (falling back to + /// an older one if the newest is already closing). Returns false if none took + /// it - the message is persisted first, so it still arrives on the next connect. pub fn send(&self, user_id: &str, msg: String) -> bool { - match self.map.get(user_id) { - Some(tx) => tx.send(msg).is_ok(), - None => false, + let Some(conns) = self.map.get(user_id) else { + return false; + }; + let mut msg = msg; + for tx in conns.iter().rev() { + match tx.send(msg) { + Ok(()) => return true, + Err(SendError(back)) => msg = back, + } } + false + } +} + +#[cfg(test)] +mod tests { + use super::Online; + use tokio::sync::mpsc::unbounded_channel; + + const U: &str = "px_00000000000000000000000000000001"; + + #[test] + fn pushes_go_to_the_newest_connection() { + let online = Online::new(); + let (a, mut ra) = unbounded_channel(); + let (b, mut rb) = unbounded_channel(); + online.insert(U, a); + online.insert(U, b); + assert!(online.send(U, "m".into())); + assert_eq!(rb.try_recv().unwrap(), "m"); + assert!(ra.try_recv().is_err(), "only the newest is live"); + } + + #[test] + fn an_older_connection_closing_keeps_the_newer_live() { + let online = Online::new(); + let (a, _ra) = unbounded_channel(); + let (b, mut rb) = unbounded_channel(); + online.insert(U, a.clone()); + online.insert(U, b); + online.remove(U, &a); + assert!(online.send(U, "m".into())); + assert_eq!(rb.try_recv().unwrap(), "m"); + } + + #[test] + fn the_newest_closing_hands_delivery_back() { + let online = Online::new(); + let (a, mut ra) = unbounded_channel(); + let (b, _rb) = unbounded_channel(); + online.insert(U, a); + online.insert(U, b.clone()); + online.remove(U, &b); + assert!(online.send(U, "m".into())); + assert_eq!(ra.try_recv().unwrap(), "m"); + } + + #[test] + fn a_dead_newest_falls_through_and_the_last_close_clears_the_entry() { + let online = Online::new(); + let (a, mut ra) = unbounded_channel(); + let (b, rb) = unbounded_channel(); + online.insert(U, a.clone()); + online.insert(U, b.clone()); + drop(rb); // newest socket already gone, not yet removed + assert!(online.send(U, "m".into())); + assert_eq!(ra.try_recv().unwrap(), "m"); + + online.remove(U, &b); + online.remove(U, &a); + assert!(!online.send(U, "m".into()), "offline once every connection closed"); + assert!(online.map.is_empty(), "no entry left behind"); } } diff --git a/server/tests/integration.rs b/server/tests/integration.rs index c3e2561..76e4b18 100644 --- a/server/tests/integration.rs +++ b/server/tests/integration.rs @@ -1198,6 +1198,66 @@ async fn server_end_to_end() { "should receive a heartbeat ping" ); + // Several connections for one account (tabs / devices): live pushes go to the + // newest; an OLDER one closing must not cut the newer off, and the newest + // closing hands live delivery back to one still open (ws/state.rs). + { + let push = |label: &str| { + let mut c = vec![0u8; 64]; + getrandom::getrandom(&mut c).unwrap(); + let body = b64.encode(&c); + let (http, base, token, to) = (http.clone(), base.clone(), token.clone(), bob.user_id.clone()); + let label = label.to_string(); + async move { + let r = http + .post(format!("{base}/messages/send")) + .header("X-Privex-Auth", &token) + .json(&serde_json::json!({ "recipient_id": to, "content": body })) + .send() + .await + .unwrap(); + assert!(r.status().is_success(), "send {label}"); + body + } + }; + let settle = || tokio::time::sleep(std::time::Duration::from_millis(300)); + // Bounded: read_until_message keeps answering pings, so a push that never + // comes would otherwise wait forever instead of failing. + let within = std::time::Duration::from_secs(10); + let t_older = ws_ticket(&http, &base, &bob_token).await; + let mut older = ws_connect(addr, &t_older).await.expect("older connects"); + let t_newer = ws_ticket(&http, &base, &bob_token).await; + let mut newer = ws_connect(addr, &t_newer).await.expect("newer connects"); + settle().await; + + older.close(None).await.unwrap(); + drop(older); + settle().await; + let sent = push("after the older closed").await; + let got = tokio::time::timeout(within, read_until_message(&mut newer)) + .await + .expect("newer got nothing live after the older closed"); + assert_eq!(got["content"].as_str().unwrap(), sent, "newer still live after the older closed"); + let ack = |m: &serde_json::Value| { + WsMessage::Text(serde_json::json!({ "type": "ack", "message_ids": [m["message_id"]] }).to_string()) + }; + newer.send(ack(&got)).await.unwrap(); + + let t_newest = ws_ticket(&http, &base, &bob_token).await; + let mut newest = ws_connect(addr, &t_newest).await.expect("newest connects"); + settle().await; + newest.close(None).await.unwrap(); + drop(newest); + settle().await; + let sent = push("after the newest closed").await; + let got = tokio::time::timeout(within, read_until_message(&mut newer)) + .await + .expect("nothing handed back live after the newest closed"); + assert_eq!(got["content"].as_str().unwrap(), sent, "delivery handed back after the newest closed"); + newer.send(ack(&got)).await.unwrap(); + settle().await; + } + // ===== Session 11: KT log + key management ===== let kt_pub = SigningKey::from_bytes(&[9u8; 32]).verifying_key(); let count_opks = |uid: String| { From 5850ea7d588620b33aac8a10bf8e5797d4853b0b Mon Sep 17 00:00:00 2001 From: sonix aka privex Date: Sun, 27 Sep 2026 18:02:50 +0530 Subject: [PATCH 2/2] fix(server): unregister a closing socket before stopping its writer Between writer.abort() and the removal, Online::send could still hand a push to the closing connection (its receiver was alive) and the aborted writer then dropped it unwritten - skipping the account's still-open connection. Remove first, then abort, so pushes in that window go to the next open connection. Co-Authored-By: Claude Opus 5.5 --- server/src/ws/handler.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/ws/handler.rs b/server/src/ws/handler.rs index d07da14..408c1a3 100644 --- a/server/src/ws/handler.rs +++ b/server/src/ws/handler.rs @@ -220,9 +220,11 @@ async fn handle_socket(socket: WebSocket, st: AppState, user_id: String) { } } - writer.abort(); - // Only THIS connection: a newer one for the account must stay live. + // Only THIS connection: a newer one for the account must stay live. Removed + // BEFORE the writer stops, so no push is accepted here and then dropped + // unwritten - it goes to the account's next open connection instead. st.online.remove(&user_id, &tx); + writer.abort(); } fn base64_content(bytes: &[u8]) -> String {